This commit is contained in:
William Mantly 2016-05-29 13:43:04 -04:00
parent 42ea77720a
commit 37fae2e413

View File

@ -17,7 +17,10 @@ var isCheckingWorkers = false;
var dopletNewID = 0; var dopletNewID = 0;
var checkDroplet = function(id, time){ var workers = (function(){
var workers = [];
workers.checkDroplet = function(id, time){
time = time || 5000; time = time || 5000;
doapi.dropletInfo(id, function(data){ doapi.dropletInfo(id, function(data){
var worker = JSON.parse(data)['droplet']; var worker = JSON.parse(data)['droplet'];
@ -25,42 +28,96 @@ var checkDroplet = function(id, time){
console.log('Droplet is now active, starting runners in 20 seconds') console.log('Droplet is now active, starting runners in 20 seconds')
setTimeout(function(worker){ setTimeout(function(worker){
console.log('Ready to start runners!') console.log('Ready to start runners!')
startRunners(workers[workers.push(makeWorkerObj(worker))-1]) workers.startRunners(workers[workers.push(makeWorkerObj(worker))-1])
isCheckingWorkers = false; isCheckingWorkers = false;
}, 20000, worker); }, 20000, worker);
return true; return true;
}else{ }else{
console.log('Worker not ready, check again in ', time, 'MS'); console.log('Worker not ready, check again in ', time, 'MS');
setTimeout(function(){ setTimeout(function(){
checkDroplet(id) workers.checkDroplet(id);
}, time); }, time);
} }
}); });
}; };
var workerCreate = function(){ workers.create = function(){
doapi.dropletCreate({ doapi.dropletCreate({
name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''),
image: '17575764' image: '17575764'
}, function(data){ }, function(data){
data = JSON.parse(data); data = JSON.parse(data);
setTimeout(function(dopletNewID){ setTimeout(function(dopletNewID){
checkDroplet(dopletNewID) workers.checkDroplet(dopletNewID);
}, 60000, data.droplet.id); }, 60000, data.droplet.id);
doapi.dropletSetTag('clworker', data.droplet.id, function(d){ doapi.dropletSetTag('clworker', data.droplet.id, function(){});
console.log(d)
});
}); });
}; };
var workerDestroy = function(worker){ workers.destroy = function(worker){
worker = worker || workers.pop(); worker = worker || workers.pop();
doapi.dropletDestroy(worker.id, function(){}); doapi.dropletDestroy(worker.id, function(){});
checkWorkersBalance(); workers.checkBalance();
}; };
var checkWorkersBalance = function(){ workers.makeWorkerObj = function(worker){
worker.networks.v4.forEach(function(value){
worker[value.type+'IP'] = value.ip_address;
});
worker.availrunners = [];
worker.ip = worker.privateIP;
worker.usedrunner = 0;
worker.index = workers.length,
worker.getRunner = function(){
if(this.availrunners.length === 0) return false;
console.log('geting runner from ', worker.name, ' aval length ', this.availrunners.length);
var runner = this.availrunners.pop();
this.usedrunner++;
label2runner[runner.label] = runner;
return runner;
}
return worker;
};
workers.destroyOld = function(){
doapi.dropletsByTag('clworker', function(data){
data = JSON.parse(data);
data['droplets'].forEach(function(worker){
console.log('found old droplet, killing it');
doapi.dropletDestroy(worker.id, function(){});
});
workers.checkBalance();
});
};
workers.startRunners = function(worker, stopPercent){
console.log('starting runners on', worker.name)
stopPercent = stopPercent || 80;
ramPercentUsed(worker.ip, function(usedMemPercent){
if(usedMemPercent < stopPercent ){
var name = 'crunner-'+(Math.random()*100).toString().replace('.','');
return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
if( !data.ip ) return setTimeout(startRunners(worker),0);
console.log('started runner')
worker.availrunners.push({
ip: data.ip,
name: name,
worker: worker,
label: worker.name + ':' + name
});
return setTimeout(startRunners(worker, stopPercent), 0);
});
}else{
setTimeout(workers.checkBalance, 10000);
console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name);
}
});
};
workers.checkBalance = function(){
if(isCheckingWorkers) return false; if(isCheckingWorkers) return false;
isCheckingWorkers = true; isCheckingWorkers = true;
var changed = false; var changed = false;
@ -68,21 +125,21 @@ var checkWorkersBalance = function(){
if(workers.length < 2){ if(workers.length < 2){
console.log('less then 2 workers, starting a droplet'); console.log('less then 2 workers, starting a droplet');
for(var i=2; i--;) workerCreate(); for(var i=2; i--;) workers.workerCreate();
return ; return ;
} }
if(workers[workers.length-1].usedrunner !== 0){ if(workers[workers.length-1].usedrunner !== 0){
console.log('last droplet has no free runners, starting droplet'); console.log('last droplet has no free runners, starting droplet');
return workerCreate(); return workers.workerCreate();
} }
if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){
console.log('Last 2 runners not used, killing last runner'); console.log('Last 2 runners not used, killing last runner');
workerDestroy(); workers.workerDestroy();
} }
for(let worker of workers){ for(let worker of workers){
if(worker.availrunners.length === 0 && worker.usedrunner === 0){ if(worker.availrunners.length === 0 && worker.usedrunner === 0){
workerDestroy(worker); workers.workerDestroy(worker);
changed = true; changed = true;
} }
} }
@ -90,11 +147,13 @@ var checkWorkersBalance = function(){
console.log('stopping workers balancing check'); console.log('stopping workers balancing check');
isCheckingWorkers = false; isCheckingWorkers = false;
if(changed) setTimeout(function(){ if(changed) setTimeout(function(){
checkWorkersBalance(); workers.checkBalance();
}, 3000); }, 3000);
}; };
// var start return workers;
})();
var ramPercentUsed = function(ip, callback){ var ramPercentUsed = function(ip, callback){
@ -113,7 +172,7 @@ var runnerFree = function(runner){
} }
delete label2runner[runner.label]; delete label2runner[runner.label];
startRunners(runner.worker); workers.startRunners(runner.worker);
}; };
var lxcTimeout = function(runner, time){ var lxcTimeout = function(runner, time){
@ -152,37 +211,6 @@ var run = function(req, res, runner){
}); });
}; };
var makeWorkerObj = function(worker){
worker.networks.v4.forEach(function(value){
worker[value.type+'IP'] = value.ip_address;
});
worker.availrunners = [];
worker.ip = worker.privateIP;
worker.usedrunner = 0;
worker.index = workers.length,
worker.getRunner = function(){
if(this.availrunners.length === 0) return false;
console.log('geting runner from ', worker.name, ' aval length ', this.availrunners.length);
var runner = this.availrunners.pop();
this.usedrunner++;
label2runner[runner.label] = runner;
return runner;
}
return worker;
};
var initWorkers = function(){
doapi.dropletsByTag('clworker', function(data){
data = JSON.parse(data);
data['droplets'].forEach(function(worker){
console.log('found old droplet, killing it');
doapi.dropletDestroy(worker.id, function(){});
});
checkWorkersBalance();
});
};
var getAvailrunner = function(runner){ var getAvailrunner = function(runner){
for(let worker of workers){ for(let worker of workers){
console.log('checking ', worker.name, ' with ', worker.availrunners.length, ' free workers'); console.log('checking ', worker.name, ' with ', worker.availrunners.length, ' free workers');
@ -195,31 +223,6 @@ var getAvailrunner = function(runner){
return false; return false;
}; };
var startRunners = function(worker, stopPercent){
console.log('starting runners on', worker.name)
stopPercent = stopPercent || 80;
ramPercentUsed(worker.ip, function(usedMemPercent){
if(usedMemPercent < stopPercent ){
var name = 'crunner-'+(Math.random()*100).toString().replace('.','');
return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
if( !data.ip ) return setTimeout(startRunners(worker),0);
console.log('started runner')
worker.availrunners.push({
ip: data.ip,
name: name,
worker: worker,
label: worker.name + ':' + name
});
return setTimeout(startRunners(worker, stopPercent), 0);
});
}else{
setTimeout(checkWorkersBalance, 10000);
console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name);
}
});
};
initWorkers(); initWorkers();
// router.get('/start/:name', function(req, res, next){ // router.get('/start/:name', function(req, res, next){