diff --git a/doapi.js b/doapi.js
index cf14911..cda8088 100644
--- a/doapi.js
+++ b/doapi.js
@@ -77,6 +77,28 @@ api = function(key){
});
}
+ this.dropletToActive = function(args){
+ var doapi = this;
+ this.dropletCreate(args, function(data){
+ data = JSON.parse(data);
+ setTimeout(function(id, args, doapi){
+ time = args.time || 10000;
+
+ doapi.dropletInfo(id, function check(data){
+ var droplet = JSON.parse(data)['droplet'];
+ if(droplet.status == 'active'){
+
+ return args.onActive(worker);
+ }else{
+ setTimeout(function(check){
+ checkDroplet(id);
+ }, time, check, .droplet.id);
+ }
+ }, 70000, data.droplet.id, args, doapi);
+ });
+ });
+ }
+
this.dropletDestroy = function(dropletID, callback){
var options = {
url: this.BASEURL+'droplets/'+dropletID,
diff --git a/routes/api.js b/routes/api.js
index c1db888..f8769c3 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -7,85 +7,60 @@ var request = require('request');
var lxc = require('../lxc');
var doapi = require('../doapi')();
-var workerSnapID = 'V1'
-
-// console.log = function(){};
-
var label2runner = {};
-var isCheckingWorkers = false;
var workers = (function(){
var workers = [];
-
- workers.currentCreatingMax = 2;
+ var worker.settings = require('./workers.json');
workers.currentCreating = 0;
-
- workers.checkDroplet = function(id, time){
- time = time || 10000;
-
- return doapi.dropletInfo(id, function(data){
- var worker = JSON.parse(data)['droplet'];
- if(worker.status == 'active'){
- // console.log('Droplet is now active, starting runners in 20 seconds');
-
- return setTimeout(function(worker){
- // console.log('Ready to start runners!');
- workers.startRunners(workers.makeWorkerObj(worker), true);
- workers.currentCreating--;
- }, 20000, worker);
- }else{
- // console.log('Worker not ready, check again in ', time, 'MS');
-
- return setTimeout(function(){
- workers.checkDroplet(id);
- }, time);
- }
- });
- };
workers.create = function(){
- if(workers.currentCreating > workers.currentCreatingMax ) return false;
- return doapi.dropletCreate({
+ if(workers.currentCreating > workers.settins.max ) return false;
+ workers.currentCreating++;
+ return doapi.dropletToActive({
name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4),
image: '18473675',
- size: '2gb'
- }, function(data){
- data = JSON.parse(data);
- workers.currentCreating++;
- setTimeout(function(dopletNewID){
- return workers.checkDroplet(dopletNewID);
- }, 70000, data.droplet.id);
+ size: '2gb',
+ onActive = function(worker){
+ doapi.domianAddRecord({
+ domain: "codeland.us",
+ type: "A",
+ name: "*."+args.worker.name+".workers",
+ data: args.worker.publicIP
+ }, function(){});
- return doapi.dropletSetTag('clworker', data.droplet.id, function(){});
+ workers.startRunners({
+ worker: workers.makeWorkerObj(worker),
+ onDone: function(worker){
+ workers.push(args.worker)
+ }
+ }),
+ workers.currentCreating--;
+ }
});
};
- workers.destroy = function(worker){
- var worker = worker || workers.pop();
- return doapi.dropletDestroy(worker.id, function(body) {
- console.log('body of destroy', body)
- });
- };
-
workers.makeWorkerObj = function(worker){
worker.networks.v4.forEach(function(value){
worker[value.type+'IP'] = value.ip_address;
});
+
worker.availrunners = [];
worker.ip = worker.privateIP;
worker.usedrunner = 0;
- worker.index = workers.length,
+ worker.index = workers.length;
+ worker.snapShotId = worker.settings.snapShotId;
+
worker.getRunner = function(){
if(this.availrunners.length === 0) return false;
// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
var runner = this.availrunners.pop();
this.usedrunner++;
- lxcTimeout(runner);
- label2runner[runner.label] = runner;
-
+ runnerTimeout(runner);
+
return runner;
}
@@ -99,88 +74,106 @@ var workers = (function(){
};
- workers.destroyOld = function(){
- var currentIDs = workers.__workersId()
- doapi.dropletsByTag('clworker', function(data){
+ workers.destroy = function(worker){
+ var worker = worker || workers.pop();
+ return doapi.dropletDestroy(worker.id, function(body) {
+ console.log('body of destroy', body);
+ });
+ };
+
+ workers.destroyOld = function(tag){
+ if(droplets.length === 0) return true;
+ var droplet = droplets.pop();
+ tag = tag || 'clworker';
+ var currentIDs = workers.__workersId();
+
+ var deleteDroplets = function(droplets){
+ console.log('checking', droplet.name, droplet.id);
+ if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
+
+ doapi.dropletDestroy(droplet.id, function(body){
+ console.log('delete body', body);
+ setTimeout(deleteDroplets(droplets));
+ });
+ }
+
+ doapi.dropletsByTag(tag, function(data){
data = JSON.parse(data);
- console.log('current worker ids', currentIDs)
+ console.log('current worker ids', currentIDs);
console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){
return item.name+'| '+item.id;
}));
- data['droplets'].forEach(function(worker){
- console.log('checking', worker.name, worker.id)
- if(~currentIDs.indexOf(worker.id)) return false;
- console.log('found old droplet, killing it');
- setTimeout(function(id) {
- // body...
- doapi.dropletDestroy(id, function(body){console.log('delete body', body)});
- }, Math.floor(Math.random() * 9) + 1, worker.id)
+ deleteDroplets(data['droplets']);
+ });
+ };
+
+ workers.startRunners = function(args){
+ // console.log('starting runners on', worker.name, worker.ip)
+ args.stopPercent = stopPercent || 80;
+ args.onStart = args.onStart || function(){};
+ args.onDone = args.onDone || function(){};
+
+ ramPercentUsed(args.worker.ip, function(usedMemPercent){
+ if(usedMemPercent > stopPercent ){
+ console.log('using', String(usedMemPercent),
+ 'percent memory, stopping runner creation!', args.worker.availrunners.length,
+ 'created on ', args.worker.name
+ );
+ args.onDone(worker)
+ return ;
+ }
+
+ var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
+ lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
+ if(!data.ip) return setTimeout(workers.startRunners,0, args);
+ // console.log('started runner on', args.worker.name)
+ args.onStart(args.worker.availrunners[args.worker.availrunners.length-1])
+
+ args.worker.availrunners.push({
+ ip: data.ip,
+ name: name,
+ worker: worker,
+ label: args.worker.name + ':' + name
+ });
+
+ setTimeout(workers.startRunners(args), 0);
});
});
};
- workers.startRunners = function(worker, newWorker, stopPercent){
- // console.log('starting runners on', worker.name, worker.ip)
- stopPercent = stopPercent || 80;
- ramPercentUsed(worker.ip, function(usedMemPercent){
- if(usedMemPercent < stopPercent ){
- var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
- return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
- if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0);
- // console.log('started runner on', worker.name)
- if(newWorker){
- worker = workers[workers.push(worker)-1];
- doapi.domianAddRecord({
- domain: "codeland.us",
- type: "A",
- name: "*."+worker.name+".workers",
- data: worker.publicIP
- }, function(){});
- }
-
- worker.availrunners.push({
- ip: data.ip,
- name: name,
- worker: worker,
- label: worker.name + ':' + name
- });
- return setTimeout(workers.startRunners(worker, false ,stopPercent), 0);
- });
- }else{
- console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name);
- }
- });
- };
-
workers.checkBalance = function(){
-
- var minWorkers = 1;
console.log('checking balance');
- if(workers.length < minWorkers){
- console.log('less then 3 workers, starting a droplet');
- for(var i=minWorkers-workers.length; i--;) workers.create();
+ if(workers.length < workers.settings.min){
+ console.log('less then '+ workers.settings.min +' workers, starting a droplet');
+ for(var i=workers.settings.min-workers.length; i--;) workers.create();
return ;
}
- // if(workers[workers.length-3].usedrunner !== 0 && workers[workers.length-2].usedrunner !== 0 && workers[workers.length-1].usedrunner !== 0){
- // console.log('last 3 workers have no free runners, starting droplet');
- // return workers.create();
- // }
- // if(workers.length > minWorkers && workers[workers.length-3].usedrunner === 0 && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0){
- // console.log('Last 2 runners not used, killing last runner', workers.length);
- // return workers.destroy();
- // }
- // for(let worker of workers){
- // if(worker.length <= 3) break;
- // if(worker.availrunners.length === 0 && worker.usedrunner === 0){
- // console.log('found zombie worker, destroying')
- // workers.destroy(worker);
- // }
- // }
+ if(minAvail > 0) return ;
- console.log('stopping workers balancing check');
+ var lastMinAval
+ for(let worker of workers.slice(-wokers.settings.minAvail)){
+ if(worker.usedrunner !== 0){
+ console.log('last 3 workers have no free runners, starting droplet');
+ return workers.create();
+ }
+ if(workers.length > workers.settings.min && workers.usedrunner === 0 ){
+ lastMinAval++;
+ }
+ }
+ if(lastMinAval == wokers.settings.minAvail){
+ console.log('Last 3 runners not used, killing last runner', workers.length);
+ return workers.destroy();
+ }
+
+ for(let worker of workers){
+ if(worker.availrunners.length === 0 && worker.usedrunner === 0){
+ console.log('found zombie worker, destroying')
+ workers.destroy(worker);
+ }
+ }
};
return workers;
@@ -196,18 +189,7 @@ var ramPercentUsed = function(ip, callback){
);
};
-var runnerFree = function(runner){
- lxc.stop(runner.name, runner.worker.ip);
- runner.worker.usedrunner--;
- if(runner.hasOwnProperty('timeout')){
- clearTimeout(runner.timeout);
- }
- delete label2runner[runner.label];
-
- workers.startRunners(runner.worker);
-};
-
-var lxcTimeout = function(runner, time){
+var runnerTimeout = function(runner, time){
time = time || 60000; // 1 minutes
if(runner.hasOwnProperty('timeout')){
@@ -219,15 +201,27 @@ var lxcTimeout = function(runner, time){
}, time);
};
+var runnerFree = function(runner){
+ lxc.stop(runner.name, runner.worker.ip);
+ runner.worker.usedrunner--;
+ if(runner.hasOwnProperty('timeout')){
+ clearTimeout(runner.timeout);
+ }
+ delete label2runner[runner.label];
+
+ workers.startRunners(runner.worker);
+};
+
var getAvailrunner = function(runner){
for(let worker of workers){
if(worker.availrunners.length === 0) continue;
if(runner && runner.worker.index <= worker.index) break;
if(runner) runnerFree(runner);
+
return worker.getRunner();
}
+
if(runner) return runner;
- return null;
};
var run = function(req, res, runner, count){
@@ -261,83 +255,26 @@ var run = function(req, res, runner, count){
// console.log('runner response:', arguments)
if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count);
body = JSON.parse(body);
+ res.json(body);
+
+ if(req.params.once) return runnerFree(runner);
+ label2runner[runner.label] = runner;
body['ip'] = runner.label;
body['rname'] = runner.name;
body['wname'] = runner.worker.name;
- lxcTimeout(runner);
- return res.json(body);
-
+ runnerTimeout(runner);
});
};
setTimeout(function(){
// console.log('Starting balance checking in 30 seconds')
setInterval(workers.checkBalance, 15000);
-}, 180000);
-
-setInterval(function(argument) {
- workers.destroyOld();
-}, 3600000); // 1 hour
+}, 180000);
workers.destroyOld();
workers.checkBalance();
-// router.get('/start/:name', function(req, res, next){
-// return lxc.start(req.params.name, function(data){
-// if(!data){
-// return res.json({
-// status: 500,
-// name: req.params.name,
-// message: data
-// });
-// }else{
-// res.json({});
-// }
-// });
-// });
-
-// router.get('/live/:template/:name', function(req, res, next){
-// return lxc.startEphemeral(req.params.name, req.params.template, function (data) {
-// console.log('live', arguments);
-// return res.json(data);
-// });
-// });
-
-// router.get('/clone/:template/:name', function(req, res, next){
-// return lxc.clone(req.params.name, req.params.template, function(data){
-// console.log('clone', arguments);
-// if( data.match(/Created runner/) ){
-// return res.json({status: 200});
-// }else{
-// return res.json({status: 500, message: data});
-// }
-// });
-// });
-
-// router.get('/destroy/:name', function(req, res, next){
-// return lxc.destroy(req.params.name, function(data){
-// console.log('destroy', arguments);
-// if(data){
-// return res.json({status: 500, message: data});
-// }else{
-// return res.json({status: 200});
-// }
-// });
-// });
-
-// router.get('/info/:name', function(req, res, next){
-// return lxc.info(req.params.name, function(data){
-// return res.json(data);
-// });
-// });
-
-// router.get('/list', function(req, res, next) {
-// return lxc.list(workers.clworker0.ip, function(data){
-// return res.json(data);
-// });
-// });
-
router.get('/stop/:name', function(req, res, next){
return lxc.stop(req.params.name, function(data){
@@ -354,14 +291,23 @@ router.get('/destroyOld', function(req, res, next) {
res.send('?');
});
+router.post('/updateID', function(req, res, next){
+
+});
+
router.get('/liststuff', function(req, res, next){
var obj = util.inspect(workers, {depth: 4});
res.send("Workers
"+obj+"
label2runner
"+util.inspect(label2runner)+'
DO calls
'+doapi.calls);
});
-router.post('/run/:ip?', function doRun(req, res, next){
+runner.get('/ping/:runner', function(req, res, next){
+ runnerTimeout(runner);
+ res.json({res:''});
+});
+
+router.post('/run/:runner?', function (req, res, next){
console.log('hit runner route');
- var runner = getAvailrunner(label2runner[req.params.ip]);
+ var runner = getAvailrunner(label2runner[req.params.runner]);
return run(req, res, runner);
});
diff --git a/routes/workers.json b/routes/workers.json
new file mode 100644
index 0000000..15eec4b
--- /dev/null
+++ b/routes/workers.json
@@ -0,0 +1 @@
+{"snapShotId":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1}
\ No newline at end of file