From 7f041355051cf08f50b54b7e9ba864538abeb452 Mon Sep 17 00:00:00 2001 From: william mantly Date: Fri, 15 Jul 2016 17:25:33 -0400 Subject: [PATCH 1/7] fu --- routes/api.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routes/api.js b/routes/api.js index 826a58d..c1db888 100644 --- a/routes/api.js +++ b/routes/api.js @@ -263,6 +263,8 @@ var run = function(req, res, runner, count){ body = JSON.parse(body); body['ip'] = runner.label; + body['rname'] = runner.name; + body['wname'] = runner.worker.name; lxcTimeout(runner); return res.json(body); From f525c4d98fa63272544b3614c29f78b6e4578eee Mon Sep 17 00:00:00 2001 From: william Date: Mon, 18 Jul 2016 02:41:55 -0400 Subject: [PATCH 2/7] omg... --- doapi.js | 22 +++ routes/api.js | 336 +++++++++++++++++++------------------------- routes/workers.json | 1 + 3 files changed, 164 insertions(+), 195 deletions(-) create mode 100644 routes/workers.json diff --git a/doapi.js b/doapi.js index cf14911..cda8088 100644 --- a/doapi.js +++ b/doapi.js @@ -77,6 +77,28 @@ api = function(key){ }); } + this.dropletToActive = function(args){ + var doapi = this; + this.dropletCreate(args, function(data){ + data = JSON.parse(data); + setTimeout(function(id, args, doapi){ + time = args.time || 10000; + + doapi.dropletInfo(id, function check(data){ + var droplet = JSON.parse(data)['droplet']; + if(droplet.status == 'active'){ + + return args.onActive(worker); + }else{ + setTimeout(function(check){ + checkDroplet(id); + }, time, check, .droplet.id); + } + }, 70000, data.droplet.id, args, doapi); + }); + }); + } + this.dropletDestroy = function(dropletID, callback){ var options = { url: this.BASEURL+'droplets/'+dropletID, diff --git a/routes/api.js b/routes/api.js index c1db888..f8769c3 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,85 +7,60 @@ var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); -var workerSnapID = 'V1' - -// console.log = function(){}; - var label2runner = {}; -var isCheckingWorkers = false; var workers = (function(){ var workers = []; - - workers.currentCreatingMax = 2; + var worker.settings = require('./workers.json'); workers.currentCreating = 0; - - workers.checkDroplet = function(id, time){ - time = time || 10000; - - return doapi.dropletInfo(id, function(data){ - var worker = JSON.parse(data)['droplet']; - if(worker.status == 'active'){ - // console.log('Droplet is now active, starting runners in 20 seconds'); - - return setTimeout(function(worker){ - // console.log('Ready to start runners!'); - workers.startRunners(workers.makeWorkerObj(worker), true); - workers.currentCreating--; - }, 20000, worker); - }else{ - // console.log('Worker not ready, check again in ', time, 'MS'); - - return setTimeout(function(){ - workers.checkDroplet(id); - }, time); - } - }); - }; workers.create = function(){ - if(workers.currentCreating > workers.currentCreatingMax ) return false; - return doapi.dropletCreate({ + if(workers.currentCreating > workers.settins.max ) return false; + workers.currentCreating++; + return doapi.dropletToActive({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4), image: '18473675', - size: '2gb' - }, function(data){ - data = JSON.parse(data); - workers.currentCreating++; - setTimeout(function(dopletNewID){ - return workers.checkDroplet(dopletNewID); - }, 70000, data.droplet.id); + size: '2gb', + onActive = function(worker){ + doapi.domianAddRecord({ + domain: "codeland.us", + type: "A", + name: "*."+args.worker.name+".workers", + data: args.worker.publicIP + }, function(){}); - return doapi.dropletSetTag('clworker', data.droplet.id, function(){}); + workers.startRunners({ + worker: workers.makeWorkerObj(worker), + onDone: function(worker){ + workers.push(args.worker) + } + }), + workers.currentCreating--; + } }); }; - workers.destroy = function(worker){ - var worker = worker || workers.pop(); - return doapi.dropletDestroy(worker.id, function(body) { - console.log('body of destroy', body) - }); - }; - workers.makeWorkerObj = function(worker){ worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); + worker.availrunners = []; worker.ip = worker.privateIP; worker.usedrunner = 0; - worker.index = workers.length, + worker.index = workers.length; + worker.snapShotId = worker.settings.snapShotId; + worker.getRunner = function(){ if(this.availrunners.length === 0) return false; // console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length); var runner = this.availrunners.pop(); this.usedrunner++; - lxcTimeout(runner); - label2runner[runner.label] = runner; - + runnerTimeout(runner); + return runner; } @@ -99,88 +74,106 @@ var workers = (function(){ }; - workers.destroyOld = function(){ - var currentIDs = workers.__workersId() - doapi.dropletsByTag('clworker', function(data){ + workers.destroy = function(worker){ + var worker = worker || workers.pop(); + return doapi.dropletDestroy(worker.id, function(body) { + console.log('body of destroy', body); + }); + }; + + workers.destroyOld = function(tag){ + if(droplets.length === 0) return true; + var droplet = droplets.pop(); + tag = tag || 'clworker'; + var currentIDs = workers.__workersId(); + + var deleteDroplets = function(droplets){ + console.log('checking', droplet.name, droplet.id); + if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets); + + doapi.dropletDestroy(droplet.id, function(body){ + console.log('delete body', body); + setTimeout(deleteDroplets(droplets)); + }); + } + + doapi.dropletsByTag(tag, function(data){ data = JSON.parse(data); - console.log('current worker ids', currentIDs) + console.log('current worker ids', currentIDs); console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){ return item.name+'| '+item.id; })); - data['droplets'].forEach(function(worker){ - console.log('checking', worker.name, worker.id) - if(~currentIDs.indexOf(worker.id)) return false; - console.log('found old droplet, killing it'); - setTimeout(function(id) { - // body... - doapi.dropletDestroy(id, function(body){console.log('delete body', body)}); - }, Math.floor(Math.random() * 9) + 1, worker.id) + deleteDroplets(data['droplets']); + }); + }; + + workers.startRunners = function(args){ + // console.log('starting runners on', worker.name, worker.ip) + args.stopPercent = stopPercent || 80; + args.onStart = args.onStart || function(){}; + args.onDone = args.onDone || function(){}; + + ramPercentUsed(args.worker.ip, function(usedMemPercent){ + if(usedMemPercent > stopPercent ){ + console.log('using', String(usedMemPercent), + 'percent memory, stopping runner creation!', args.worker.availrunners.length, + 'created on ', args.worker.name + ); + args.onDone(worker) + return ; + } + + var name = 'crunner-'+(Math.random()*100).toString().slice(-4); + lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){ + if(!data.ip) return setTimeout(workers.startRunners,0, args); + // console.log('started runner on', args.worker.name) + args.onStart(args.worker.availrunners[args.worker.availrunners.length-1]) + + args.worker.availrunners.push({ + ip: data.ip, + name: name, + worker: worker, + label: args.worker.name + ':' + name + }); + + setTimeout(workers.startRunners(args), 0); }); }); }; - workers.startRunners = function(worker, newWorker, stopPercent){ - // console.log('starting runners on', worker.name, worker.ip) - stopPercent = stopPercent || 80; - ramPercentUsed(worker.ip, function(usedMemPercent){ - if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().slice(-4); - return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); - // console.log('started runner on', worker.name) - if(newWorker){ - worker = workers[workers.push(worker)-1]; - doapi.domianAddRecord({ - domain: "codeland.us", - type: "A", - name: "*."+worker.name+".workers", - data: worker.publicIP - }, function(){}); - } - - worker.availrunners.push({ - ip: data.ip, - name: name, - worker: worker, - label: worker.name + ':' + name - }); - return setTimeout(workers.startRunners(worker, false ,stopPercent), 0); - }); - }else{ - console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); - } - }); - }; - workers.checkBalance = function(){ - - var minWorkers = 1; console.log('checking balance'); - if(workers.length < minWorkers){ - console.log('less then 3 workers, starting a droplet'); - for(var i=minWorkers-workers.length; i--;) workers.create(); + if(workers.length < workers.settings.min){ + console.log('less then '+ workers.settings.min +' workers, starting a droplet'); + for(var i=workers.settings.min-workers.length; i--;) workers.create(); return ; } - // if(workers[workers.length-3].usedrunner !== 0 && workers[workers.length-2].usedrunner !== 0 && workers[workers.length-1].usedrunner !== 0){ - // console.log('last 3 workers have no free runners, starting droplet'); - // return workers.create(); - // } - // if(workers.length > minWorkers && workers[workers.length-3].usedrunner === 0 && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0){ - // console.log('Last 2 runners not used, killing last runner', workers.length); - // return workers.destroy(); - // } - // for(let worker of workers){ - // if(worker.length <= 3) break; - // if(worker.availrunners.length === 0 && worker.usedrunner === 0){ - // console.log('found zombie worker, destroying') - // workers.destroy(worker); - // } - // } + if(minAvail > 0) return ; - console.log('stopping workers balancing check'); + var lastMinAval + for(let worker of workers.slice(-wokers.settings.minAvail)){ + if(worker.usedrunner !== 0){ + console.log('last 3 workers have no free runners, starting droplet'); + return workers.create(); + } + if(workers.length > workers.settings.min && workers.usedrunner === 0 ){ + lastMinAval++; + } + } + if(lastMinAval == wokers.settings.minAvail){ + console.log('Last 3 runners not used, killing last runner', workers.length); + return workers.destroy(); + } + + for(let worker of workers){ + if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + console.log('found zombie worker, destroying') + workers.destroy(worker); + } + } }; return workers; @@ -196,18 +189,7 @@ var ramPercentUsed = function(ip, callback){ ); }; -var runnerFree = function(runner){ - lxc.stop(runner.name, runner.worker.ip); - runner.worker.usedrunner--; - if(runner.hasOwnProperty('timeout')){ - clearTimeout(runner.timeout); - } - delete label2runner[runner.label]; - - workers.startRunners(runner.worker); -}; - -var lxcTimeout = function(runner, time){ +var runnerTimeout = function(runner, time){ time = time || 60000; // 1 minutes if(runner.hasOwnProperty('timeout')){ @@ -219,15 +201,27 @@ var lxcTimeout = function(runner, time){ }, time); }; +var runnerFree = function(runner){ + lxc.stop(runner.name, runner.worker.ip); + runner.worker.usedrunner--; + if(runner.hasOwnProperty('timeout')){ + clearTimeout(runner.timeout); + } + delete label2runner[runner.label]; + + workers.startRunners(runner.worker); +}; + var getAvailrunner = function(runner){ for(let worker of workers){ if(worker.availrunners.length === 0) continue; if(runner && runner.worker.index <= worker.index) break; if(runner) runnerFree(runner); + return worker.getRunner(); } + if(runner) return runner; - return null; }; var run = function(req, res, runner, count){ @@ -261,83 +255,26 @@ var run = function(req, res, runner, count){ // console.log('runner response:', arguments) if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count); body = JSON.parse(body); + res.json(body); + + if(req.params.once) return runnerFree(runner); + label2runner[runner.label] = runner; body['ip'] = runner.label; body['rname'] = runner.name; body['wname'] = runner.worker.name; - lxcTimeout(runner); - return res.json(body); - + runnerTimeout(runner); }); }; setTimeout(function(){ // console.log('Starting balance checking in 30 seconds') setInterval(workers.checkBalance, 15000); -}, 180000); - -setInterval(function(argument) { - workers.destroyOld(); -}, 3600000); // 1 hour +}, 180000); workers.destroyOld(); workers.checkBalance(); -// router.get('/start/:name', function(req, res, next){ -// return lxc.start(req.params.name, function(data){ -// if(!data){ -// return res.json({ -// status: 500, -// name: req.params.name, -// message: data -// }); -// }else{ -// res.json({}); -// } -// }); -// }); - -// router.get('/live/:template/:name', function(req, res, next){ -// return lxc.startEphemeral(req.params.name, req.params.template, function (data) { -// console.log('live', arguments); -// return res.json(data); -// }); -// }); - -// router.get('/clone/:template/:name', function(req, res, next){ -// return lxc.clone(req.params.name, req.params.template, function(data){ -// console.log('clone', arguments); -// if( data.match(/Created runner/) ){ -// return res.json({status: 200}); -// }else{ -// return res.json({status: 500, message: data}); -// } -// }); -// }); - -// router.get('/destroy/:name', function(req, res, next){ -// return lxc.destroy(req.params.name, function(data){ -// console.log('destroy', arguments); -// if(data){ -// return res.json({status: 500, message: data}); -// }else{ -// return res.json({status: 200}); -// } -// }); -// }); - -// router.get('/info/:name', function(req, res, next){ -// return lxc.info(req.params.name, function(data){ -// return res.json(data); -// }); -// }); - -// router.get('/list', function(req, res, next) { -// return lxc.list(workers.clworker0.ip, function(data){ -// return res.json(data); -// }); -// }); - router.get('/stop/:name', function(req, res, next){ return lxc.stop(req.params.name, function(data){ @@ -354,14 +291,23 @@ router.get('/destroyOld', function(req, res, next) { res.send('?'); }); +router.post('/updateID', function(req, res, next){ + +}); + router.get('/liststuff', function(req, res, next){ var obj = util.inspect(workers, {depth: 4}); res.send("

Workers

"+obj+"

label2runner

"+util.inspect(label2runner)+'

DO calls

'+doapi.calls); }); -router.post('/run/:ip?', function doRun(req, res, next){ +runner.get('/ping/:runner', function(req, res, next){ + runnerTimeout(runner); + res.json({res:''}); +}); + +router.post('/run/:runner?', function (req, res, next){ console.log('hit runner route'); - var runner = getAvailrunner(label2runner[req.params.ip]); + var runner = getAvailrunner(label2runner[req.params.runner]); return run(req, res, runner); }); diff --git a/routes/workers.json b/routes/workers.json new file mode 100644 index 0000000..15eec4b --- /dev/null +++ b/routes/workers.json @@ -0,0 +1 @@ +{"snapShotId":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file From ec565c2803fe97f8f9f09a518052c427613c7cb4 Mon Sep 17 00:00:00 2001 From: william mantly Date: Wed, 20 Jul 2016 15:36:07 -0400 Subject: [PATCH 3/7] done --- doapi.js | 42 +++++++++---- lxc.js | 2 +- routes/api.js | 149 +++++++++++++++++++++++++++++--------------- routes/workers.json | 2 +- testAPI.py | 5 +- 5 files changed, 135 insertions(+), 65 deletions(-) diff --git a/doapi.js b/doapi.js index cda8088..a91177d 100644 --- a/doapi.js +++ b/doapi.js @@ -34,6 +34,7 @@ api = function(key){ }; this.dropletSetTag = function(tag, dropletID, callback) { + callback = callback || function(){}; var data = { resources: [ { @@ -55,6 +56,8 @@ api = function(key){ }; this.dropletCreate = function(args, callback){ + callback = callback || function(){}; + var data = { name: args.name, // || return false, region: args.region || 'nyc3', @@ -79,27 +82,30 @@ api = function(key){ this.dropletToActive = function(args){ var doapi = this; + args.onCreated = args.onCreate || function(){}; + this.dropletCreate(args, function(data){ data = JSON.parse(data); - setTimeout(function(id, args, doapi){ + args.onCreate(data, args); + setTimeout(function check(id, args, doapi){ time = args.time || 10000; - - doapi.dropletInfo(id, function check(data){ + doapi.dropletInfo(id, function (data){ var droplet = JSON.parse(data)['droplet']; if(droplet.status == 'active'){ - return args.onActive(worker); + return args.onActive(droplet, args); }else{ - setTimeout(function(check){ - checkDroplet(id); - }, time, check, .droplet.id); + setTimeout(function(check, id){ + check(id, args, doapi); + }, time, check, droplet.id); } - }, 70000, data.droplet.id, args, doapi); - }); + }); + }, 70000, data.droplet.id, args, doapi); }); - } + }; this.dropletDestroy = function(dropletID, callback){ + callback = callback || function(){}; var options = { url: this.BASEURL+'droplets/'+dropletID, headers: this.headers @@ -123,6 +129,20 @@ api = function(key){ }); }; + this.tagCreate = function(tag, callback){ + callback = callback || function(){}; + var options = { + url: this.BASEURL+'tags', + headers: this.headers, + body: JSON.stringify({name: tag}) + }; + this.calls++; + + return request.post(options, function(error, response, body){ + return callback(body, response, error); + }); + }; + this.tagsList = function(callback){ var options = { url: this.BASEURL+'tags', @@ -136,7 +156,7 @@ api = function(key){ }; this.domianAddRecord = function(args, callback){ - + callback = callback || function(){}; var options = { url: this.BASEURL+'domains/'+ args.domain +'/records', headers: this.headers, diff --git a/lxc.js b/lxc.js index 9bf3796..dbd9ec0 100644 --- a/lxc.js +++ b/lxc.js @@ -4,7 +4,7 @@ var exec = require('child_process').exec; function sysExec(command, ip, callback){ ip = ip || '104.236.77.157'; command = new Buffer(command).toString('base64') - command = 'ssh -o StrictHostKeyChecking=no virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; + command = 'ssh -i ~/.ssh/clw_rsa -o StrictHostKeyChecking=no virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ diff --git a/routes/api.js b/routes/api.js index f8769c3..a2b8dc2 100644 --- a/routes/api.js +++ b/routes/api.js @@ -11,29 +11,35 @@ var label2runner = {}; var workers = (function(){ var workers = []; - var worker.settings = require('./workers.json'); + workers.settings = require('./workers.json'); workers.currentCreating = 0; workers.create = function(){ - if(workers.currentCreating > workers.settins.max ) return false; + if(workers.currentCreating > workers.settings.max ) return false; workers.currentCreating++; - return doapi.dropletToActive({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4), - image: '18473675', - size: '2gb', - onActive = function(worker){ - doapi.domianAddRecord({ - domain: "codeland.us", - type: "A", - name: "*."+args.worker.name+".workers", - data: args.worker.publicIP - }, function(){}); - + return doapi.dropletToActive({ + name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4), + image: workers.settings.image, + size: workers.settings.size, + onCreate: function(data){ + doapi.dropletSetTag('clw'+workers.settings.version, data.droplet.id); + }, + onActive: function(worker, args){ workers.startRunners({ worker: workers.makeWorkerObj(worker), - onDone: function(worker){ - workers.push(args.worker) + onStart: function(runner, args){ + workers.push(args.worker); + doapi.domianAddRecord({ + domain: "codeland.us", + type: "A", + name: "*."+worker.name+".workers", + data: worker.publicIP + }); + args.onStart = function(){}; + }, + onDone: function(args){ + console.log("done with workers"); } }), workers.currentCreating--; @@ -48,10 +54,9 @@ var workers = (function(){ }); worker.availrunners = []; - worker.ip = worker.privateIP; + worker.ip = worker.publicIP; worker.usedrunner = 0; worker.index = workers.length; - worker.snapShotId = worker.settings.snapShotId; worker.getRunner = function(){ if(this.availrunners.length === 0) return false; @@ -64,6 +69,10 @@ var workers = (function(){ return runner; } + worker.freeAll = function(){ + this.availrunners.forEach(runnerFree); + } + return worker; }; @@ -82,18 +91,15 @@ var workers = (function(){ }; workers.destroyOld = function(tag){ - if(droplets.length === 0) return true; - var droplet = droplets.pop(); - tag = tag || 'clworker'; + tag = tag || 'clw'+workers.settings.version; var currentIDs = workers.__workersId(); - var deleteDroplets = function(droplets){ - console.log('checking', droplet.name, droplet.id); + if(droplets.length === 0) return true; + var droplet = droplets.pop(); if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets); doapi.dropletDestroy(droplet.id, function(body){ - console.log('delete body', body); - setTimeout(deleteDroplets(droplets)); + setTimeout(deleteDroplets, 1000, droplets); }); } @@ -101,7 +107,7 @@ var workers = (function(){ data = JSON.parse(data); console.log('current worker ids', currentIDs); console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){ - return item.name+'| '+item.id; + return item.name+' | '+item.id; })); deleteDroplets(data['droplets']); @@ -109,35 +115,38 @@ var workers = (function(){ }; workers.startRunners = function(args){ - // console.log('starting runners on', worker.name, worker.ip) - args.stopPercent = stopPercent || 80; + // console.log('starting runners on', args.worker.name, args.worker.ip) + args.stopPercent = args.stopPercent || 80; args.onStart = args.onStart || function(){}; args.onDone = args.onDone || function(){}; ramPercentUsed(args.worker.ip, function(usedMemPercent){ - if(usedMemPercent > stopPercent ){ + if(usedMemPercent > args.stopPercent ){ console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', args.worker.availrunners.length, 'created on ', args.worker.name ); - args.onDone(worker) + args.onDone(args) return ; } var name = 'crunner-'+(Math.random()*100).toString().slice(-4); + // console.log('Free ram check passed!') lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){ - if(!data.ip) return setTimeout(workers.startRunners,0, args); + if(!data.ip) return setTimeout(workers.startRunners, 0, args); // console.log('started runner on', args.worker.name) - args.onStart(args.worker.availrunners[args.worker.availrunners.length-1]) - args.worker.availrunners.push({ + var runner = { ip: data.ip, name: name, - worker: worker, + worker: args.worker, label: args.worker.name + ':' + name - }); + }; + args.onStart(runner, args) - setTimeout(workers.startRunners(args), 0); + args.worker.availrunners.push(runner); + + setTimeout(workers.startRunners, 0, args); }); }); }; @@ -151,7 +160,7 @@ var workers = (function(){ return ; } - if(minAvail > 0) return ; + if(workers.settings.minAvail > 0) return ; var lastMinAval for(let worker of workers.slice(-wokers.settings.minAvail)){ @@ -176,6 +185,11 @@ var workers = (function(){ } }; + workers.concat = function(newWorkers){ + newWorkers.forEach(workers.push) + } + + doapi.tagCreate('clw'+workers.settings.version) return workers; })(); @@ -196,9 +210,7 @@ var runnerTimeout = function(runner, time){ clearTimeout(runner.timeout); } - return runner.timeout = setTimeout(function(){ - runnerFree(runner); - }, time); + return runner.timeout = setTimeout(runnerFree, time, runner); }; var runnerFree = function(runner){ @@ -209,7 +221,7 @@ var runnerFree = function(runner){ } delete label2runner[runner.label]; - workers.startRunners(runner.worker); + workers.startRunners({worker: runner.worker}); }; var getAvailrunner = function(runner){ @@ -234,10 +246,9 @@ var run = function(req, res, runner, count){ return res.json({error: 'No runners, try again soon.'}); } - if(count > 3){ + if(count > 2){ console.log('to many reties on runner'); - res.status(400); - return res.json({error: 'Runner restarted to many times'}); + return res.status(400).json({error: 'Runner restarted to many times'}); } var httpOptions = { @@ -250,19 +261,22 @@ var run = function(req, res, runner, count){ }) }; - return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count); body = JSON.parse(body); - res.json(body); - - if(req.params.once) return runnerFree(runner); + + if(req.query.once){ + res.json(body); + return runnerFree(runner, 0); + } label2runner[runner.label] = runner; body['ip'] = runner.label; body['rname'] = runner.name; body['wname'] = runner.worker.name; + res.json(body); + runnerTimeout(runner); }); }; @@ -270,7 +284,7 @@ var run = function(req, res, runner, count){ setTimeout(function(){ // console.log('Starting balance checking in 30 seconds') setInterval(workers.checkBalance, 15000); -}, 180000); +}, 600000); workers.destroyOld(); workers.checkBalance(); @@ -292,7 +306,42 @@ router.get('/destroyOld', function(req, res, next) { }); router.post('/updateID', function(req, res, next){ + var newWorkers = []; + var newWorkersTarget = workers.length; + var newWokerImage = req.query.image; + for(var i = 0; iWorkers
"+obj+"

label2runner

"+util.inspect(label2runner)+'

DO calls

'+doapi.calls); }); -runner.get('/ping/:runner', function(req, res, next){ +router.get('/ping/:runner', function(req, res, next){ runnerTimeout(runner); res.json({res:''}); }); diff --git a/routes/workers.json b/routes/workers.json index 15eec4b..fea18f4 100644 --- a/routes/workers.json +++ b/routes/workers.json @@ -1 +1 @@ -{"snapShotId":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file +{"image":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file diff --git a/testAPI.py b/testAPI.py index e507afd..ac507f9 100644 --- a/testAPI.py +++ b/testAPI.py @@ -7,8 +7,9 @@ def testAPI(times=100, sleep=2): for i in range(times): try: res = r.post( - 'http://codeland.bytedev.co:2000/api/run', - data={'code': 'pwd'} + 'http://127.0.0.1:2000/api/run', + data={'code': 'pwd'}, + params={'once': True} ) if res.status_code != 200: errors += 1 print(i, res.status_code, res.content) From bfa3e689b3ac4b356b72a322b2d67f31ab2ec824 Mon Sep 17 00:00:00 2001 From: william mantly Date: Thu, 21 Jul 2016 23:45:21 -0400 Subject: [PATCH 4/7] upgrade works --- doapi.js | 13 +++-- package.json | 5 +- routes/api.js | 132 ++++++++++++++++++++++++++++---------------- routes/workers.json | 2 +- workers.json | 9 +++ 5 files changed, 104 insertions(+), 57 deletions(-) create mode 100644 workers.json diff --git a/doapi.js b/doapi.js index a91177d..47f41ad 100644 --- a/doapi.js +++ b/doapi.js @@ -81,26 +81,29 @@ api = function(key){ } this.dropletToActive = function(args){ - var doapi = this; + args.__doapi = this; // hold the DO api in the agrs scope args.onCreated = args.onCreate || function(){}; this.dropletCreate(args, function(data){ data = JSON.parse(data); args.onCreate(data, args); - setTimeout(function check(id, args, doapi){ + + // check if the server is ready, giving time to allow + // digital ocean to do its thing + setTimeout(function check(id, args){ time = args.time || 10000; - doapi.dropletInfo(id, function (data){ + args.__doapi.dropletInfo(id, function (data){ var droplet = JSON.parse(data)['droplet']; if(droplet.status == 'active'){ return args.onActive(droplet, args); }else{ setTimeout(function(check, id){ - check(id, args, doapi); + check(id, args); }, time, check, droplet.id); } }); - }, 70000, data.droplet.id, args, doapi); + }, 70000, data.droplet.id, args); }); }; diff --git a/package.json b/package.json index b1becda..46613a0 100644 --- a/package.json +++ b/package.json @@ -11,10 +11,11 @@ "debug": "~2.2.0", "ejs": "~2.3.3", "express": "~4.13.1", + "jsonfile": "^2.3.1", "morgan": "~1.6.1", - "serve-favicon": "~2.3.0", "node.extend": "~1.1.5", "redis": "~2.4.2", - "request": "~2.69.0" + "request": "~2.69.0", + "serve-favicon": "~2.3.0" } } diff --git a/routes/api.js b/routes/api.js index a2b8dc2..e3882ff 100644 --- a/routes/api.js +++ b/routes/api.js @@ -4,6 +4,7 @@ var express = require('express'); var router = express.Router(); var util = require('util'); var request = require('request'); +var jsonfile = require('jsonfile'); var lxc = require('../lxc'); var doapi = require('../doapi')(); @@ -18,12 +19,12 @@ var workers = (function(){ workers.create = function(){ if(workers.currentCreating > workers.settings.max ) return false; workers.currentCreating++; - return doapi.dropletToActive({ + doapi.dropletToActive({ name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4), image: workers.settings.image, size: workers.settings.size, onCreate: function(data){ - doapi.dropletSetTag('clw'+workers.settings.version, data.droplet.id); + doapi.dropletSetTag('clwV'+workers.settings.version, data.droplet.id); }, onActive: function(worker, args){ workers.startRunners({ @@ -41,7 +42,7 @@ var workers = (function(){ onDone: function(args){ console.log("done with workers"); } - }), + }); workers.currentCreating--; } }); @@ -64,15 +65,10 @@ var workers = (function(){ var runner = this.availrunners.pop(); this.usedrunner++; runnerTimeout(runner); - return runner; } - worker.freeAll = function(){ - this.availrunners.forEach(runnerFree); - } - return worker; }; @@ -91,7 +87,7 @@ var workers = (function(){ }; workers.destroyOld = function(tag){ - tag = tag || 'clw'+workers.settings.version; + tag = tag || 'clwV'+workers.settings.version; var currentIDs = workers.__workersId(); var deleteDroplets = function(droplets){ if(droplets.length === 0) return true; @@ -116,13 +112,17 @@ var workers = (function(){ workers.startRunners = function(args){ // console.log('starting runners on', args.worker.name, args.worker.ip) - args.stopPercent = args.stopPercent || 80; + if(!args.worker || workers.settings.image > args.worker.image.id){ + console.log('blocked outdated worker', workers.settings.image, args.worker.image.id) + return ; + } + args.stopPercent = args.stopPercent || 20; args.onStart = args.onStart || function(){}; args.onDone = args.onDone || function(){}; ramPercentUsed(args.worker.ip, function(usedMemPercent){ if(usedMemPercent > args.stopPercent ){ - console.log('using', String(usedMemPercent), + console.log('using', String(usedMemPercent).trim(), 'percent memory, stopping runner creation!', args.worker.availrunners.length, 'created on ', args.worker.name ); @@ -160,9 +160,18 @@ var workers = (function(){ return ; } + for(let worker of workers){ + console.log("checking", worker.name, "if zombie") + if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + workers.splice(workers.indexOf(worker), 1) + console.log('found zombie worker, destroying'); + workers.destroy(worker); + } + } + if(workers.settings.minAvail > 0) return ; - var lastMinAval + var lastMinAval = 0; for(let worker of workers.slice(-wokers.settings.minAvail)){ if(worker.usedrunner !== 0){ console.log('last 3 workers have no free runners, starting droplet'); @@ -177,19 +186,20 @@ var workers = (function(){ return workers.destroy(); } - for(let worker of workers){ - if(worker.availrunners.length === 0 && worker.usedrunner === 0){ - console.log('found zombie worker, destroying') - workers.destroy(worker); - } - } + }; + workers.settingsSave = function(){ + jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) { + console.error(err) + }); }; - workers.concat = function(newWorkers){ - newWorkers.forEach(workers.push) - } + workers.add = function(newWorkers){ + newWorkers.forEach(function(worker){ + workers.push(worker); + }); + }; - doapi.tagCreate('clw'+workers.settings.version) + doapi.tagCreate('clwV'+workers.settings.version); return workers; })(); @@ -306,38 +316,62 @@ router.get('/destroyOld', function(req, res, next) { }); router.post('/updateID', function(req, res, next){ - var newWorkers = []; - var newWorkersTarget = workers.length; - var newWokerImage = req.query.image; - for(var i = 0; i= args.newWorkers.target){ + console.log('upgrade complete!') + workers.settings.image = args.newWorkers.image; + workers.settings.size = args.newWorkers.size; - workers.concat(newWorkers); + workers.forEach(function(worker){ + worker.availrunners.forEach(function(runner){ + lxc.stop(runner.name, runner.worker.ip); + }); + worker.availrunners = []; + }); + + workers.add(args.newWorkers.workers); + workers.settingsSave(); + workers.checkBalance(); } } - }) + + }); } }); } diff --git a/routes/workers.json b/routes/workers.json index fea18f4..231c92a 100644 --- a/routes/workers.json +++ b/routes/workers.json @@ -1 +1 @@ -{"image":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file +{"image":"17641214","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file diff --git a/workers.json b/workers.json new file mode 100644 index 0000000..d92e9e3 --- /dev/null +++ b/workers.json @@ -0,0 +1,9 @@ +{ + "image": "18473675", + "version": 5, + "lastSnapShotId": "0", + "size": "2gb", + "max": 100, + "min": 1, + "minAvail": 1 +} From 00f68ef6490eac2a486dee3c8e7fb0258ffd39e1 Mon Sep 17 00:00:00 2001 From: william mantly Date: Sat, 23 Jul 2016 01:40:14 -0400 Subject: [PATCH 5/7] works with settings file change --- routes/api.js | 24 +++++++++++++++--------- workers.json | 4 ++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/routes/api.js b/routes/api.js index e3882ff..6bf73c6 100644 --- a/routes/api.js +++ b/routes/api.js @@ -126,7 +126,7 @@ var workers = (function(){ 'percent memory, stopping runner creation!', args.worker.availrunners.length, 'created on ', args.worker.name ); - args.onDone(args) + args.onDone(args); return ; } @@ -142,7 +142,7 @@ var workers = (function(){ worker: args.worker, label: args.worker.name + ':' + name }; - args.onStart(runner, args) + args.onStart(runner, args); args.worker.availrunners.push(runner); @@ -161,9 +161,9 @@ var workers = (function(){ } for(let worker of workers){ - console.log("checking", worker.name, "if zombie") + console.log("checking", worker.name, "if zombie"); if(worker.availrunners.length === 0 && worker.usedrunner === 0){ - workers.splice(workers.indexOf(worker), 1) + workers.splice(workers.indexOf(worker), 1); console.log('found zombie worker, destroying'); workers.destroy(worker); } @@ -189,7 +189,7 @@ var workers = (function(){ }; workers.settingsSave = function(){ jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) { - console.error(err) + console.error(err); }); }; @@ -318,13 +318,17 @@ router.get('/destroyOld', function(req, res, next) { router.post('/updateID', function(req, res, next){ var newWorkers = { workers: [], - target: workers.length, image: req.query.image, + target: req.query.target || workers.length, size: req.query.size || workers.settings.size, - version: workers.settings.version+1 + version: workers.settings.version+1, + min: req.query.min || workers.settings, + minAvail: req.query.minAvail || workers.settings }; doapi.tagCreate('clwV'+newWorkers.version); + workers.destroyOld('clwV'+newWorkers.version); + for(var i=0; i Date: Sat, 23 Jul 2016 01:41:53 -0400 Subject: [PATCH 6/7] another commit --- lxc.js | 1 + workers.json | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lxc.js b/lxc.js index dbd9ec0..ab544d1 100644 --- a/lxc.js +++ b/lxc.js @@ -1,6 +1,7 @@ 'use strict'; var exec = require('child_process').exec; + function sysExec(command, ip, callback){ ip = ip || '104.236.77.157'; command = new Buffer(command).toString('base64') diff --git a/workers.json b/workers.json index cc86c04..2589c94 100644 --- a/workers.json +++ b/workers.json @@ -1,9 +1,8 @@ { "image": "18607562", "version": 5, - "lastSnapShotId": "0", "size": "512mb", "max": 100, - "min": 1, - "minAvail": 1 + "min": 3, + "minAvail": 3 } From ef2e9533661d7cf970dd3d01ed1ae3457165d838 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 22 Mar 2017 21:10:03 -0400 Subject: [PATCH 7/7] stuff --- app.js | 6 ++++++ routes/api.js | 4 +++- routes/workers.json | 2 +- workers.json | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app.js b/app.js index 5555393..1cbeaf3 100644 --- a/app.js +++ b/app.js @@ -18,6 +18,12 @@ app.use(bodyParser.urlencoded({ extended: false })); app.use(cookieParser()); app.use(express.static(path.join(__dirname, 'public'))); +app.use(function(req, res, next) { + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + next(); +}); + app.use('/', require('./routes/index')); app.use('/api', require('./routes/api')); diff --git a/routes/api.js b/routes/api.js index 6bf73c6..b33029a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -24,6 +24,7 @@ var workers = (function(){ image: workers.settings.image, size: workers.settings.size, onCreate: function(data){ + console.log(data) doapi.dropletSetTag('clwV'+workers.settings.version, data.droplet.id); }, onActive: function(worker, args){ @@ -116,7 +117,7 @@ var workers = (function(){ console.log('blocked outdated worker', workers.settings.image, args.worker.image.id) return ; } - args.stopPercent = args.stopPercent || 20; + args.stopPercent = args.stopPercent || 80; args.onStart = args.onStart || function(){}; args.onDone = args.onDone || function(){}; @@ -390,6 +391,7 @@ router.get('/liststuff', function(req, res, next){ }); router.get('/ping/:runner', function(req, res, next){ + var runner = label2runner[req.params.runner]; runnerTimeout(runner); res.json({res:''}); }); diff --git a/routes/workers.json b/routes/workers.json index 231c92a..1de1734 100644 --- a/routes/workers.json +++ b/routes/workers.json @@ -1 +1 @@ -{"image":"17641214","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file +{"image":"23505298","version":9,"lastSnapShotId":"18670778","size":"512mb","max":100,"min":3,"minAvail":3} diff --git a/workers.json b/workers.json index 2589c94..0a6e3c2 100644 --- a/workers.json +++ b/workers.json @@ -1,6 +1,6 @@ { - "image": "18607562", - "version": 5, + "image": "23469057", + "version": 8, "size": "512mb", "max": 100, "min": 3,