wmantly/lxc_manager_node
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

omg...

Browse Source
This commit is contained in:
William Mantly 2016-07-18 02:41:55 -04:00
parent 7f04135505
commit f525c4d98f
3 changed files with 164 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
doapi.js
View File

@ -77,6 +77,28 @@ api = function(key){
}); });
} }
this.dropletToActive = function(args){
var doapi = this;
this.dropletCreate(args, function(data){
data = JSON.parse(data);
setTimeout(function(id, args, doapi){
time = args.time || 10000;
doapi.dropletInfo(id, function check(data){
var droplet = JSON.parse(data)['droplet'];
if(droplet.status == 'active'){
return args.onActive(worker);
}else{
setTimeout(function(check){
checkDroplet(id);
}, time, check, .droplet.id);
}
}, 70000, data.droplet.id, args, doapi);
});
});
}
this.dropletDestroy = function(dropletID, callback){ this.dropletDestroy = function(dropletID, callback){
var options = { var options = {
url: this.BASEURL+'droplets/'+dropletID, url: this.BASEURL+'droplets/'+dropletID,

324
routes/api.js
View File

@ -7,83 +7,58 @@ var request = require('request');
var lxc = require('../lxc'); var lxc = require('../lxc');
var doapi = require('../doapi')(); var doapi = require('../doapi')();
var workerSnapID = 'V1'
// console.log = function(){};
var label2runner = {}; var label2runner = {};
var isCheckingWorkers = false;
var workers = (function(){ var workers = (function(){
var workers = []; var workers = [];
var worker.settings = require('./workers.json');
workers.currentCreatingMax = 2;
workers.currentCreating = 0; workers.currentCreating = 0;
workers.checkDroplet = function(id, time){
time = time || 10000;
return doapi.dropletInfo(id, function(data){
var worker = JSON.parse(data)['droplet'];
if(worker.status == 'active'){
// console.log('Droplet is now active, starting runners in 20 seconds');
return setTimeout(function(worker){
// console.log('Ready to start runners!');
workers.startRunners(workers.makeWorkerObj(worker), true);
workers.currentCreating--;
}, 20000, worker);
}else{
// console.log('Worker not ready, check again in ', time, 'MS');
return setTimeout(function(){
workers.checkDroplet(id);
}, time);
}
});
};
workers.create = function(){ workers.create = function(){
if(workers.currentCreating > workers.currentCreatingMax ) return false; if(workers.currentCreating > workers.settins.max ) return false;
return doapi.dropletCreate({ workers.currentCreating++;
return doapi.dropletToActive({
name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4), name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4),
image: '18473675', image: '18473675',
size: '2gb' size: '2gb',
}, function(data){ onActive = function(worker){
data = JSON.parse(data); doapi.domianAddRecord({
workers.currentCreating++; domain: "codeland.us",
setTimeout(function(dopletNewID){ type: "A",
return workers.checkDroplet(dopletNewID); name: "*."+args.worker.name+".workers",
}, 70000, data.droplet.id); data: args.worker.publicIP
}, function(){});
return doapi.dropletSetTag('clworker', data.droplet.id, function(){}); workers.startRunners({
worker: workers.makeWorkerObj(worker),
onDone: function(worker){
workers.push(args.worker)
}
}),
workers.currentCreating--;
}
}); });
}; };
workers.destroy = function(worker){
var worker = worker || workers.pop();
return doapi.dropletDestroy(worker.id, function(body) {
console.log('body of destroy', body)
});
};
workers.makeWorkerObj = function(worker){ workers.makeWorkerObj = function(worker){
worker.networks.v4.forEach(function(value){ worker.networks.v4.forEach(function(value){
worker[value.type+'IP'] = value.ip_address; worker[value.type+'IP'] = value.ip_address;
}); });
worker.availrunners = []; worker.availrunners = [];
worker.ip = worker.privateIP; worker.ip = worker.privateIP;
worker.usedrunner = 0; worker.usedrunner = 0;
worker.index = workers.length, worker.index = workers.length;
worker.snapShotId = worker.settings.snapShotId;
worker.getRunner = function(){ worker.getRunner = function(){
if(this.availrunners.length === 0) return false; if(this.availrunners.length === 0) return false;
// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length); // console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
var runner = this.availrunners.pop(); var runner = this.availrunners.pop();
this.usedrunner++; this.usedrunner++;
lxcTimeout(runner); runnerTimeout(runner);
label2runner[runner.label] = runner;
return runner; return runner;
@ -99,88 +74,106 @@ var workers = (function(){
}; };
workers.destroyOld = function(){ workers.destroy = function(worker){
var currentIDs = workers.__workersId() var worker = worker || workers.pop();
doapi.dropletsByTag('clworker', function(data){ return doapi.dropletDestroy(worker.id, function(body) {
data = JSON.parse(data); console.log('body of destroy', body);
console.log('current worker ids', currentIDs)
console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){
return item.name+'| '+item.id;
}));
data['droplets'].forEach(function(worker){
console.log('checking', worker.name, worker.id)
if(~currentIDs.indexOf(worker.id)) return false;
console.log('found old droplet, killing it');
setTimeout(function(id) {
// body...
doapi.dropletDestroy(id, function(body){console.log('delete body', body)});
}, Math.floor(Math.random() * 9) + 1, worker.id)
});
}); });
}; };
workers.startRunners = function(worker, newWorker, stopPercent){ workers.destroyOld = function(tag){
// console.log('starting runners on', worker.name, worker.ip) if(droplets.length === 0) return true;
stopPercent = stopPercent || 80; var droplet = droplets.pop();
ramPercentUsed(worker.ip, function(usedMemPercent){ tag = tag || 'clworker';
if(usedMemPercent < stopPercent ){ var currentIDs = workers.__workersId();
var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ var deleteDroplets = function(droplets){
if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('checking', droplet.name, droplet.id);
// console.log('started runner on', worker.name) if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
if(newWorker){
worker = workers[workers.push(worker)-1]; doapi.dropletDestroy(droplet.id, function(body){
doapi.domianAddRecord({ console.log('delete body', body);
domain: "codeland.us", setTimeout(deleteDroplets(droplets));
type: "A", });
name: "*."+worker.name+".workers",
data: worker.publicIP
}, function(){});
} }
worker.availrunners.push({ doapi.dropletsByTag(tag, function(data){
data = JSON.parse(data);
console.log('current worker ids', currentIDs);
console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){
return item.name+'| '+item.id;
}));
deleteDroplets(data['droplets']);
});
};
workers.startRunners = function(args){
// console.log('starting runners on', worker.name, worker.ip)
args.stopPercent = stopPercent || 80;
args.onStart = args.onStart || function(){};
args.onDone = args.onDone || function(){};
ramPercentUsed(args.worker.ip, function(usedMemPercent){
if(usedMemPercent > stopPercent ){
console.log('using', String(usedMemPercent),
'percent memory, stopping runner creation!', args.worker.availrunners.length,
'created on ', args.worker.name
);
args.onDone(worker)
return ;
}
var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
if(!data.ip) return setTimeout(workers.startRunners,0, args);
// console.log('started runner on', args.worker.name)
args.onStart(args.worker.availrunners[args.worker.availrunners.length-1])
args.worker.availrunners.push({
ip: data.ip, ip: data.ip,
name: name, name: name,
worker: worker, worker: worker,
label: worker.name + ':' + name label: args.worker.name + ':' + name
}); });
return setTimeout(workers.startRunners(worker, false ,stopPercent), 0);
setTimeout(workers.startRunners(args), 0);
}); });
}else{
console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name);
}
}); });
}; };
workers.checkBalance = function(){ workers.checkBalance = function(){
var minWorkers = 1;
console.log('checking balance'); console.log('checking balance');
if(workers.length < minWorkers){ if(workers.length < workers.settings.min){
console.log('less then 3 workers, starting a droplet'); console.log('less then '+ workers.settings.min +' workers, starting a droplet');
for(var i=minWorkers-workers.length; i--;) workers.create(); for(var i=workers.settings.min-workers.length; i--;) workers.create();
return ; return ;
} }
// if(workers[workers.length-3].usedrunner !== 0 && workers[workers.length-2].usedrunner !== 0 && workers[workers.length-1].usedrunner !== 0){
// console.log('last 3 workers have no free runners, starting droplet');
// return workers.create();
// }
// if(workers.length > minWorkers && workers[workers.length-3].usedrunner === 0 && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0){
// console.log('Last 2 runners not used, killing last runner', workers.length);
// return workers.destroy();
// }
// for(let worker of workers){ if(minAvail > 0) return ;
// if(worker.length <= 3) break;
// if(worker.availrunners.length === 0 && worker.usedrunner === 0){
// console.log('found zombie worker, destroying')
// workers.destroy(worker);
// }
// }
console.log('stopping workers balancing check'); var lastMinAval
for(let worker of workers.slice(-wokers.settings.minAvail)){
if(worker.usedrunner !== 0){
console.log('last 3 workers have no free runners, starting droplet');
return workers.create();
}
if(workers.length > workers.settings.min && workers.usedrunner === 0 ){
lastMinAval++;
}
}
if(lastMinAval == wokers.settings.minAvail){
console.log('Last 3 runners not used, killing last runner', workers.length);
return workers.destroy();
}
for(let worker of workers){
if(worker.availrunners.length === 0 && worker.usedrunner === 0){
console.log('found zombie worker, destroying')
workers.destroy(worker);
}
}
}; };
return workers; return workers;
@ -196,18 +189,7 @@ var ramPercentUsed = function(ip, callback){
); );
}; };
var runnerFree = function(runner){ var runnerTimeout = function(runner, time){
lxc.stop(runner.name, runner.worker.ip);
runner.worker.usedrunner--;
if(runner.hasOwnProperty('timeout')){
clearTimeout(runner.timeout);
}
delete label2runner[runner.label];
workers.startRunners(runner.worker);
};
var lxcTimeout = function(runner, time){
time = time || 60000; // 1 minutes time = time || 60000; // 1 minutes
if(runner.hasOwnProperty('timeout')){ if(runner.hasOwnProperty('timeout')){
@ -219,15 +201,27 @@ var lxcTimeout = function(runner, time){
}, time); }, time);
}; };
var runnerFree = function(runner){
lxc.stop(runner.name, runner.worker.ip);
runner.worker.usedrunner--;
if(runner.hasOwnProperty('timeout')){
clearTimeout(runner.timeout);
}
delete label2runner[runner.label];
workers.startRunners(runner.worker);
};
var getAvailrunner = function(runner){ var getAvailrunner = function(runner){
for(let worker of workers){ for(let worker of workers){
if(worker.availrunners.length === 0) continue; if(worker.availrunners.length === 0) continue;
if(runner && runner.worker.index <= worker.index) break; if(runner && runner.worker.index <= worker.index) break;
if(runner) runnerFree(runner); if(runner) runnerFree(runner);
return worker.getRunner(); return worker.getRunner();
} }
if(runner) return runner; if(runner) return runner;
return null;
}; };
var run = function(req, res, runner, count){ var run = function(req, res, runner, count){
@ -261,13 +255,15 @@ var run = function(req, res, runner, count){
// console.log('runner response:', arguments) // console.log('runner response:', arguments)
if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count); if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count);
body = JSON.parse(body); body = JSON.parse(body);
res.json(body);
if(req.params.once) return runnerFree(runner);
label2runner[runner.label] = runner;
body['ip'] = runner.label; body['ip'] = runner.label;
body['rname'] = runner.name; body['rname'] = runner.name;
body['wname'] = runner.worker.name; body['wname'] = runner.worker.name;
lxcTimeout(runner); runnerTimeout(runner);
return res.json(body);
}); });
}; };
@ -276,68 +272,9 @@ setTimeout(function(){
setInterval(workers.checkBalance, 15000); setInterval(workers.checkBalance, 15000);
}, 180000); }, 180000);
setInterval(function(argument) {
workers.destroyOld();
}, 3600000); // 1 hour
workers.destroyOld(); workers.destroyOld();
workers.checkBalance(); workers.checkBalance();
// router.get('/start/:name', function(req, res, next){
// return lxc.start(req.params.name, function(data){
// if(!data){
// return res.json({
// status: 500,
// name: req.params.name,
// message: data
// });
// }else{
// res.json({});
// }
// });
// });
// router.get('/live/:template/:name', function(req, res, next){
// return lxc.startEphemeral(req.params.name, req.params.template, function (data) {
// console.log('live', arguments);
// return res.json(data);
// });
// });
// router.get('/clone/:template/:name', function(req, res, next){
// return lxc.clone(req.params.name, req.params.template, function(data){
// console.log('clone', arguments);
// if( data.match(/Created runner/) ){
// return res.json({status: 200});
// }else{
// return res.json({status: 500, message: data});
// }
// });
// });
// router.get('/destroy/:name', function(req, res, next){
// return lxc.destroy(req.params.name, function(data){
// console.log('destroy', arguments);
// if(data){
// return res.json({status: 500, message: data});
// }else{
// return res.json({status: 200});
// }
// });
// });
// router.get('/info/:name', function(req, res, next){
// return lxc.info(req.params.name, function(data){
// return res.json(data);
// });
// });
// router.get('/list', function(req, res, next) {
// return lxc.list(workers.clworker0.ip, function(data){
// return res.json(data);
// });
// });
router.get('/stop/:name', function(req, res, next){ router.get('/stop/:name', function(req, res, next){
return lxc.stop(req.params.name, function(data){ return lxc.stop(req.params.name, function(data){
@ -354,14 +291,23 @@ router.get('/destroyOld', function(req, res, next) {
res.send('?'); res.send('?');
}); });
router.post('/updateID', function(req, res, next){
});
router.get('/liststuff', function(req, res, next){ router.get('/liststuff', function(req, res, next){
var obj = util.inspect(workers, {depth: 4}); var obj = util.inspect(workers, {depth: 4});
res.send("<h1>Workers</h1><pre>"+obj+"</pre><h1>label2runner</h1><pre>"+util.inspect(label2runner)+'</pre><h1>DO calls</h1>'+doapi.calls); res.send("<h1>Workers</h1><pre>"+obj+"</pre><h1>label2runner</h1><pre>"+util.inspect(label2runner)+'</pre><h1>DO calls</h1>'+doapi.calls);
}); });
router.post('/run/:ip?', function doRun(req, res, next){ runner.get('/ping/:runner', function(req, res, next){
runnerTimeout(runner);
res.json({res:''});
});
router.post('/run/:runner?', function (req, res, next){
console.log('hit runner route'); console.log('hit runner route');
var runner = getAvailrunner(label2runner[req.params.ip]); var runner = getAvailrunner(label2runner[req.params.runner]);
return run(req, res, runner); return run(req, res, runner);
}); });

1
routes/workers.json Normal file
View File

@ -0,0 +1 @@
{"snapShotId":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1}