From 70bd796bc63c22b29cc421fd4834d80eb52c449a Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:08:13 -0400 Subject: [PATCH 001/175] ssh --- lxc.js | 3 ++- routes/api.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lxc.js b/lxc.js index a54a942..50de262 100644 --- a/lxc.js +++ b/lxc.js @@ -2,7 +2,8 @@ var exec = require('child_process').exec; function sysExec(command, callback){ - command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; + command = 'ssh virt@104.236.77.157 ' + command; + // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ return function(err,data,stderr){ diff --git a/routes/api.js b/routes/api.js index 5b3491e..f3e1cc2 100644 --- a/routes/api.js +++ b/routes/api.js @@ -47,6 +47,7 @@ var lxcTimeout = function(ip, time){ } timeoutEvents[name] = setTimeout(function(){ lxc.stop(name); + startAll(); }, time); } From a65c87017a54d196eb20a1b2bd22cb890dd4e56e Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:15:30 -0400 Subject: [PATCH 002/175] ssh --- routes/api.js | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/routes/api.js b/routes/api.js index f3e1cc2..6d201d5 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,22 +7,33 @@ var redis = require("redis"); var client = redis.createClient(); var request = require('request'); var lxc = require('../lxc'); -var os = require('os'); -var spawn = require('child_process').spawn; var totalMem = os.totalmem(); var timeoutEvents = {}; var ip2name = {}; var availContainers = []; -var usedContainers = []; +var usedContainers = []; + + +var exec = require('child_process').exec; + +function sysExec(command, callback){ + command = 'ssh virt@104.236.77.157 ' + command; + // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; + + return exec(command, (function(callback){ + return function(err,data,stderr){ + if(callback){ + return callback(data, err, stderr); + } + } + })(callback)); +}; var getFreeMem = function(callback){ - var prc = spawn('free', ['-b']); - - prc.stdout.setEncoding('utf8'); - prc.stdout.on('data', function (data) { + var parseFree = function (data) { var str = data.toString() var lines = str.split(/\n/g); for(var i = 0; i < lines.length; i++) { @@ -30,9 +41,9 @@ var getFreeMem = function(callback){ } var freeMem = Number(lines[2][3]); return callback(freeMem); - }); - - prc.on('close', function (code) { + } + return sysExec('free -b ', function(data) { + return parseFree(data, callback); }); }; @@ -71,15 +82,15 @@ var runner = function(req, res, ip){ var addToRedis = function(){ lxc.info(req.params.name, null, function(data){ - var domain = req.query.domain || 'vm42.us'; - domain = req.params.name+'.'+domain; - client.SADD("hosts", domain, function(){}); - - var ip = data.ip + ':5000'; - client.HSET(domain, "ip", ip, redis.print); - client.HSET(domain, "updated", (new Date).getTime(), redis.print); - client.hset(domain, "include", "proxy.include"); - return res.json({status: 200, info: data}); + var domain = req.query.domain || 'vm42.us'; + domain = req.params.name+'.'+domain; + client.SADD("hosts", domain, function(){}); + + var ip = data.ip + ':5000'; + client.HSET(domain, "ip", ip, redis.print); + client.HSET(domain, "updated", (new Date).getTime(), redis.print); + client.hset(domain, "include", "proxy.include"); + return res.json({status: 200, info: data}); }); }; From 1db7bd8f3cc713b498f99d4d67ae527e745050b4 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:25:05 -0400 Subject: [PATCH 003/175] ssh --- routes/api.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/routes/api.js b/routes/api.js index 6d201d5..3ad3e49 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,7 +42,7 @@ var getFreeMem = function(callback){ var freeMem = Number(lines[2][3]); return callback(freeMem); } - return sysExec('free -b ', function(data) { + return sysExec("python3 -c \"a=`head /proc/meminfo | grep MemAvail | grep -Po '\d+'`;t=`head /proc/meminfo | grep MemTotal | grep -Po '\d+'`;print(round(((t-a) / t)*100, 2))\"", function(data) { return parseFree(data, callback); }); }; @@ -183,10 +183,9 @@ router.post('/run/:ip?', function doRun(req, res, next){ // freeMem: 97700 totalmem 513818624 usedMem: 0 // freeMem: 420,472 totalmem 513,818,624 usedMem: 100 var startAll = function(){ - getFreeMem(function(freeMem){ - var usedMemPercent = Math.round(( (totalMem-freeMem) /totalMem)*100); - console.log('freeMem:', freeMem, 'totalmem', totalMem, 'usedMemPercent:', usedMemPercent); - if(usedMemPercent < 81 ){ + getFreeMem(function(usedMemPercent){ + + if(usedMemPercent < 85 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner', function(data){ ip2name[data.ip] = name; From 7f0489cfa92f8f27f8e63bd8a90875a9e2793fac Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:25:41 -0400 Subject: [PATCH 004/175] ssh --- routes/api.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/routes/api.js b/routes/api.js index 3ad3e49..ad70b94 100644 --- a/routes/api.js +++ b/routes/api.js @@ -33,15 +33,6 @@ function sysExec(command, callback){ var getFreeMem = function(callback){ - var parseFree = function (data) { - var str = data.toString() - var lines = str.split(/\n/g); - for(var i = 0; i < lines.length; i++) { - lines[i] = lines[i].split(/\s+/); - } - var freeMem = Number(lines[2][3]); - return callback(freeMem); - } return sysExec("python3 -c \"a=`head /proc/meminfo | grep MemAvail | grep -Po '\d+'`;t=`head /proc/meminfo | grep MemTotal | grep -Po '\d+'`;print(round(((t-a) / t)*100, 2))\"", function(data) { return parseFree(data, callback); }); From 19395652c424eea32a8ba7bb3aa66c1f4b36e6ec Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:26:07 -0400 Subject: [PATCH 005/175] ssh --- routes/api.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index ad70b94..7399e3e 100644 --- a/routes/api.js +++ b/routes/api.js @@ -8,14 +8,11 @@ var client = redis.createClient(); var request = require('request'); var lxc = require('../lxc'); - -var totalMem = os.totalmem(); var timeoutEvents = {}; var ip2name = {}; var availContainers = []; var usedContainers = []; - var exec = require('child_process').exec; function sysExec(command, callback){ From 651a4d82a596f4431f10adedb98ff8fd75e10387 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:28:59 -0400 Subject: [PATCH 006/175] ssh --- lxc.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lxc.js b/lxc.js index 50de262..7b851f0 100644 --- a/lxc.js +++ b/lxc.js @@ -2,7 +2,8 @@ var exec = require('child_process').exec; function sysExec(command, callback){ - command = 'ssh virt@104.236.77.157 ' + command; + command = new Buffer(command).toString('base64') + command = 'ssh virt@104.236.77.157 "' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ From 2fc82766342211ba7e611472a73fb3cebb638b68 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:30:18 -0400 Subject: [PATCH 007/175] ssh --- routes/api.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7399e3e..f228d27 100644 --- a/routes/api.js +++ b/routes/api.js @@ -3,8 +3,6 @@ var express = require('express'); var router = express.Router(); var extend = require('node.extend'); -var redis = require("redis"); -var client = redis.createClient(); var request = require('request'); var lxc = require('../lxc'); @@ -68,20 +66,6 @@ var runner = function(req, res, ip){ }); }; -var addToRedis = function(){ - lxc.info(req.params.name, null, function(data){ - var domain = req.query.domain || 'vm42.us'; - domain = req.params.name+'.'+domain; - client.SADD("hosts", domain, function(){}); - - var ip = data.ip + ':5000'; - client.HSET(domain, "ip", ip, redis.print); - client.HSET(domain, "updated", (new Date).getTime(), redis.print); - client.hset(domain, "include", "proxy.include"); - return res.json({status: 200, info: data}); - }); -}; - router.get('/start/:name', function(req, res, next){ return lxc.start(req.params.name, function(data){ if(!data){ From 2a6cce62961a041997c6c5f34bf4acaa161e54a7 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:31:17 -0400 Subject: [PATCH 008/175] ssh --- routes/api.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index f228d27..8d8de8c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -28,9 +28,7 @@ function sysExec(command, callback){ var getFreeMem = function(callback){ - return sysExec("python3 -c \"a=`head /proc/meminfo | grep MemAvail | grep -Po '\d+'`;t=`head /proc/meminfo | grep MemTotal | grep -Po '\d+'`;print(round(((t-a) / t)*100, 2))\"", function(data) { - return parseFree(data, callback); - }); + return sysExec("python3 -c \"a=`head /proc/meminfo | grep MemAvail | grep -Po '\d+'`;t=`head /proc/meminfo | grep MemTotal | grep -Po '\d+'`;print(round(((t-a) / t)*100, 2))\"", callback); }; From c97100c9a74c99d39cbe74a074d08da18bfe82fb Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:33:02 -0400 Subject: [PATCH 009/175] ssh --- lxc.js | 2 +- routes/api.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lxc.js b/lxc.js index 7b851f0..01aa594 100644 --- a/lxc.js +++ b/lxc.js @@ -3,7 +3,7 @@ var exec = require('child_process').exec; function sysExec(command, callback){ command = new Buffer(command).toString('base64') - command = 'ssh virt@104.236.77.157 "' + command + '|base64 --decode|bash"'; + command = 'ssh virt@104.236.77.157 "echo ' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ diff --git a/routes/api.js b/routes/api.js index 8d8de8c..f7ab1b1 100644 --- a/routes/api.js +++ b/routes/api.js @@ -14,7 +14,8 @@ var usedContainers = []; var exec = require('child_process').exec; function sysExec(command, callback){ - command = 'ssh virt@104.236.77.157 ' + command; + command = new Buffer(command).toString('base64') + command = 'ssh virt@104.236.77.157 "echo ' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ From d10c699145f3a8e13be30a95b3952fa0f1a435a7 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 30 Apr 2016 23:36:19 -0400 Subject: [PATCH 010/175] ssh --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index f7ab1b1..63c8336 100644 --- a/routes/api.js +++ b/routes/api.js @@ -158,7 +158,7 @@ var startAll = function(){ if(usedMemPercent < 85 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); - return lxc.startEphemeral(name, 'crunner', function(data){ + return lxc.startEphemeral(name, 'crunner0', function(data){ ip2name[data.ip] = name; availContainers.push(data.ip); return startAll(); From 7e5fae983bba3a3e619d2938d48d4d36c764a003 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 2 May 2016 00:40:50 -0400 Subject: [PATCH 011/175] ssh --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 63c8336..65feb83 100644 --- a/routes/api.js +++ b/routes/api.js @@ -156,7 +156,7 @@ router.post('/run/:ip?', function doRun(req, res, next){ var startAll = function(){ getFreeMem(function(usedMemPercent){ - if(usedMemPercent < 85 ){ + if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', function(data){ ip2name[data.ip] = name; From a158d05b9db08058b45c6ed01a09a870e4313ff6 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 4 May 2016 23:43:08 -0400 Subject: [PATCH 012/175] maybe --- .gitignore | 3 ++ doapi.js | 51 +++++++++++++++++++++ lxc.js | 6 ++- routes/api.js | 123 ++++++++++++++++++++++++-------------------------- 4 files changed, 118 insertions(+), 65 deletions(-) create mode 100644 doapi.js diff --git a/.gitignore b/.gitignore index aba296a..8e2fee9 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ node_modules # Debug log from npm npm-debug.log .c9 + +# keys +secrets.js \ No newline at end of file diff --git a/doapi.js b/doapi.js new file mode 100644 index 0000000..1cc5421 --- /dev/null +++ b/doapi.js @@ -0,0 +1,51 @@ +var request = require('request'); + + +api = function(key){ + if(!key){ + key = require('./secrets.js').doAPI; + } + this.url = 'https://digitalocean.com/v2/'; + this.headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer '+key + } + + this.byTag: function(tag, callback){ + var options = { + url: this.url+'droplets?tag_name='+tag, + headers: this.headers + } + return request.get(options, function(error, response, body){ + return callback(body, response, error); + }) + }; + + this.setTag: function(id, tag, callback){ + return request.post(url+) + }; +} + +api = { +} + + + + + + + +var httpOptions = { + url:'http://' + ip + ':15000', + body: JSON.stringify({ + code: req.body.code + }) + }; + + return request.post(httpOptions, function(error, response, body){ + body = JSON.parse(body); + body['ip'] = ip.replace('10.0.', ''); + return res.json(body); + }); + +module.exports = api; \ No newline at end of file diff --git a/lxc.js b/lxc.js index 01aa594..be310e6 100644 --- a/lxc.js +++ b/lxc.js @@ -1,9 +1,10 @@ 'use strict'; var exec = require('child_process').exec; -function sysExec(command, callback){ +function sysExec(command, callback, ip){ + ip = ip || '104.236.77.157'; command = new Buffer(command).toString('base64') - command = 'ssh virt@104.236.77.157 "echo ' + command + '|base64 --decode|bash"'; + command = 'ssh virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ @@ -39,6 +40,7 @@ var lxcORM = function(){ }; var lxc = { + exec: sysExec, create: function(name, template, config, callback){ return sysExec('lxc-create -n '+name+' -t '+template, callback); }, diff --git a/routes/api.js b/routes/api.js index 65feb83..0aeab83 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,52 +7,48 @@ var request = require('request'); var lxc = require('../lxc'); var timeoutEvents = {}; -var ip2name = {}; +var label2container = {}; var availContainers = []; var usedContainers = []; - -var exec = require('child_process').exec; - -function sysExec(command, callback){ - command = new Buffer(command).toString('base64') - command = 'ssh virt@104.236.77.157 "echo ' + command + '|base64 --decode|bash"'; - // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; - - return exec(command, (function(callback){ - return function(err,data,stderr){ - if(callback){ - return callback(data, err, stderr); - } - } - })(callback)); +var workers = { + clworker0: { + ip: '104.236.77.157', + name: 'clworker0' + } }; -var getFreeMem = function(callback){ +var getFreeMem = function(ip, callback){ - return sysExec("python3 -c \"a=`head /proc/meminfo | grep MemAvail | grep -Po '\d+'`;t=`head /proc/meminfo | grep MemTotal | grep -Po '\d+'`;print(round(((t-a) / t)*100, 2))\"", callback); + return lxc.sysExec( + "python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\d+'`;print(round(((t-a)/t)*100, 2))\"", + ip, + callback + ); }; - -var lxcTimeout = function(ip, time){ - var name = ip2name[ip]; - console.log(name) +var lxcTimeout = function(container, time){ time = time || 900000; // 15 minutes var keys = Object.keys(timeoutEvents) - if(keys.indexOf(name) !== -1){ - clearTimeout(timeoutEvents[name]) + + if(keys.indexOf(container.label) !== -1){ + clearTimeout(timeoutEvents[container.label]) } - timeoutEvents[name] = setTimeout(function(){ - lxc.stop(name); - startAll(); + + return timeoutEvents[container.label] = setTimeout(function(){ + lxc.stop(name, container); + return startAll(container.worker); }, time); } -var runner = function(req, res, ip){ - lxcTimeout(ip); +var runner = function(req, res, container){ + lxcTimeout(container); var httpOptions = { - url:'http://' + ip + ':15000', + url: 'http://' + container.worker.ip, + headers: { + host: container.name + }, body: JSON.stringify({ code: req.body.code }) @@ -60,15 +56,41 @@ var runner = function(req, res, ip){ return request.post(httpOptions, function(error, response, body){ body = JSON.parse(body); - body['ip'] = ip.replace('10.0.', ''); + body['ip'] = container.label; return res.json(body); }); }; +var startWorkers = function(clworker, stopPercent){ + stopPercent = stopPercent || 81; + getFreeMem(function(usedMemPercent, clworker){ + if(usedMemPercent < 81 ){ + var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); + return lxc.startEphemeral(name, 'crunner0', clworker, function(data){ + availContainers.push({ + ip: data.ip, + name: name, + worker: worker, + label: worker.name+':'+name; + }); + return startWorkers(clworker); + }); + }else{ + console.log('using', usedMemPercent, 'percent memory, stopping container creation!', availContainers.length, 'created'); + } + }); +}; + +startWorkers(worker.clworker0); + router.get('/start/:name', function(req, res, next){ return lxc.start(req.params.name, function(data){ if(!data){ - return res.json({status: 500, name: req.params.name, message: data}); + return res.json({ + status: 500, + name: req.params.name, + message: data + }); }else{ res.json({}); } @@ -132,43 +154,18 @@ router.post('/run/:ip?', function doRun(req, res, next){ return lxc.list(function(data){ if(!req.params.ip) data = []; - var ip = '10.0.'+ req.params.ip; - var found = false; + var container = label2container[req.params.ip] || null; - for(var idx=data.length; idx--;){ - if( data[idx]['ipv4'] === ip ){ - found = true; - break; - } - } - if(found){ - return runner(req, res, ip) + if(container){ + return runner(req, res, container); }else{ - return runner(req, res, availContainers.pop()); + container = availContainers.splice(0,1); + label2container[container.worker.name+':'+container.name] = container; + return runner(req, res, container); } }); }); -// freeMem: 97700 totalmem 513818624 usedMem: 0 -// freeMem: 420,472 totalmem 513,818,624 usedMem: 100 -var startAll = function(){ - getFreeMem(function(usedMemPercent){ - - if(usedMemPercent < 81 ){ - var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); - return lxc.startEphemeral(name, 'crunner0', function(data){ - ip2name[data.ip] = name; - availContainers.push(data.ip); - return startAll(); - }); - }else{ - console.log('using', usedMemPercent, 'percent memory, stopping container creation!', availContainers.length, 'created'); - } - }); -} - -startAll(); - module.exports = router; From fcb4612e1cbb968f7b5e644383dcb5cab9cd550b Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:18:10 -0400 Subject: [PATCH 013/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 0aeab83..ef5326b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -71,7 +71,7 @@ var startWorkers = function(clworker, stopPercent){ ip: data.ip, name: name, worker: worker, - label: worker.name+':'+name; + label: worker.name+':'+name }); return startWorkers(clworker); }); From 3a1276db6770b0a58f038af5d14f77d9d2b90ac7 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:18:55 -0400 Subject: [PATCH 014/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index ef5326b..2054633 100644 --- a/routes/api.js +++ b/routes/api.js @@ -81,7 +81,7 @@ var startWorkers = function(clworker, stopPercent){ }); }; -startWorkers(worker.clworker0); +startWorkers(workers.clworker0); router.get('/start/:name', function(req, res, next){ return lxc.start(req.params.name, function(data){ From 2e21a7d91bc0971cc4bd60b945b2dc7510b8beba Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:19:13 -0400 Subject: [PATCH 015/175] closer --- lxc.js | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/lxc.js b/lxc.js index be310e6..067fcb3 100644 --- a/lxc.js +++ b/lxc.js @@ -16,31 +16,9 @@ function sysExec(command, callback, ip){ })(callback)); }; -var Container = function(config){ - this.name = config.name; - this.state = config.state; - this.ip = config.ip || (config.ipv4 || '').replace('-', '') || null ; -} - -Container.prototype.autoShutDown = function(time) { - time = time || 600000; - - // this.__shutDownTimeout = setTimeout(function(){}, this.autoShutDown): -}; - -var lxcORM = function(){ - var orm = {} - lxc.list(function(data){ - for(var idx = data.length; idx--;){ - orm[data[idx].name] = new Container(data[idx]); - } - }); - - return orm -}; - var lxc = { exec: sysExec, + create: function(name, template, config, callback){ return sysExec('lxc-create -n '+name+' -t '+template, callback); }, @@ -80,8 +58,8 @@ var lxc = { }); }, - stop: function(name, callback){ - return sysExec('lxc-stop -n '+ name, callback); + stop: function(name, callback, ip){ + return sysExec('lxc-stop -n '+ name, ip, callback); }, freeze: function(name, callback){ @@ -137,6 +115,3 @@ var lxc = { }; module.exports = lxc; - -var orm = lxcORM() -setTimeout(function(){console.log(orm)}, 5000) From 18cbb5536bf95fe0419ab234e4c7ecec5abeb552 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:20:05 -0400 Subject: [PATCH 016/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 2054633..64442ad 100644 --- a/routes/api.js +++ b/routes/api.js @@ -19,7 +19,7 @@ var workers = { var getFreeMem = function(ip, callback){ - return lxc.sysExec( + return lxc.exec( "python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\d+'`;print(round(((t-a)/t)*100, 2))\"", ip, callback From 1db5cbb600e05d5bf2b672d83660386999b4b82d Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:22:44 -0400 Subject: [PATCH 017/175] closer --- lxc.js | 2 +- routes/api.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lxc.js b/lxc.js index 067fcb3..b7022c6 100644 --- a/lxc.js +++ b/lxc.js @@ -40,7 +40,7 @@ var lxc = { return sysExec('lxc-start --name '+name+' --daemon', callback); }, - startEphemeral: function(name, base_name, callback){ + startEphemeral: function(name, base_name, ip, callback){ var command = 'lxc-start-ephemeral -o '+base_name+ ' -n '+name +' --union-type overlayfs -d'; return sysExec(command, function(data){ console.log('startEphemeral', arguments); diff --git a/routes/api.js b/routes/api.js index 64442ad..4e92e3f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -66,7 +66,7 @@ var startWorkers = function(clworker, stopPercent){ getFreeMem(function(usedMemPercent, clworker){ if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); - return lxc.startEphemeral(name, 'crunner0', clworker, function(data){ + return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ availContainers.push({ ip: data.ip, name: name, From 7834e892bffaa50c9baccd5f9012c16fea63b03e Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:24:23 -0400 Subject: [PATCH 018/175] closer --- lxc.js | 2 +- routes/api.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lxc.js b/lxc.js index b7022c6..0dbc9d1 100644 --- a/lxc.js +++ b/lxc.js @@ -42,7 +42,7 @@ var lxc = { startEphemeral: function(name, base_name, ip, callback){ var command = 'lxc-start-ephemeral -o '+base_name+ ' -n '+name +' --union-type overlayfs -d'; - return sysExec(command, function(data){ + return sysExec(command, ip, function(data){ console.log('startEphemeral', arguments); if(data.match("doesn't exist.")){ return callback({status: 500, error: "doesn't exist."}); diff --git a/routes/api.js b/routes/api.js index 4e92e3f..9ec214f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -70,8 +70,8 @@ var startWorkers = function(clworker, stopPercent){ availContainers.push({ ip: data.ip, name: name, - worker: worker, - label: worker.name+':'+name + worker: clworker, + label: clworker.name+':'+name }); return startWorkers(clworker); }); From f80988a929022cbabf328fc483cbb2f9fc9e10aa Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:26:14 -0400 Subject: [PATCH 019/175] closer --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 9ec214f..6d0d7b2 100644 --- a/routes/api.js +++ b/routes/api.js @@ -67,6 +67,7 @@ var startWorkers = function(clworker, stopPercent){ if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ + console.log('worker:', clworker.name, 'name:', name) availContainers.push({ ip: data.ip, name: name, From 2b8ef1d39ef24f04cccf83af601431ac1448ee22 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:36:12 -0400 Subject: [PATCH 020/175] closer --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 6d0d7b2..b2a21bf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -63,6 +63,7 @@ var runner = function(req, res, container){ var startWorkers = function(clworker, stopPercent){ stopPercent = stopPercent || 81; + console.log(clworker) getFreeMem(function(usedMemPercent, clworker){ if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From 8b0556b545ee30d66e8653b42fb27394d55fd611 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:36:53 -0400 Subject: [PATCH 021/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index b2a21bf..99671aa 100644 --- a/routes/api.js +++ b/routes/api.js @@ -64,7 +64,7 @@ var runner = function(req, res, container){ var startWorkers = function(clworker, stopPercent){ stopPercent = stopPercent || 81; console.log(clworker) - getFreeMem(function(usedMemPercent, clworker){ + getFreeMem(clworker.ip, function(usedMemPercent, clworker){ if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ From 82ce62a67895e9d352dbabdbbbf23ba4d19b592c Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:37:45 -0400 Subject: [PATCH 022/175] closer --- lxc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lxc.js b/lxc.js index 0dbc9d1..96d4bec 100644 --- a/lxc.js +++ b/lxc.js @@ -1,7 +1,7 @@ 'use strict'; var exec = require('child_process').exec; -function sysExec(command, callback, ip){ +function sysExec(command, ip, callback){ ip = ip || '104.236.77.157'; command = new Buffer(command).toString('base64') command = 'ssh virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; From 89621817825dcb45385c45815b00863ef9de6636 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:43:48 -0400 Subject: [PATCH 023/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 99671aa..45b9436 100644 --- a/routes/api.js +++ b/routes/api.js @@ -20,7 +20,7 @@ var workers = { var getFreeMem = function(ip, callback){ return lxc.exec( - "python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\d+'`;print(round(((t-a)/t)*100, 2))\"", + "python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"", ip, callback ); From 8f9a43c7438dcc0d38d55bbf3c85449381027a5c Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:46:03 -0400 Subject: [PATCH 024/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 45b9436..c5d7efa 100644 --- a/routes/api.js +++ b/routes/api.js @@ -146,7 +146,7 @@ router.get('/info/:name', function(req, res, next){ }); router.get('/list', function(req, res, next) { - return lxc.list(function(data){ + return lxc.list(ip, function(data){ return res.json(data); }); }); From 923b6fdd7887995d3f3def69eccd254c3763794c Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 00:57:32 -0400 Subject: [PATCH 025/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index c5d7efa..0cabb25 100644 --- a/routes/api.js +++ b/routes/api.js @@ -146,7 +146,7 @@ router.get('/info/:name', function(req, res, next){ }); router.get('/list', function(req, res, next) { - return lxc.list(ip, function(data){ + return lxc.list(workers.clworker0.ip, function(data){ return res.json(data); }); }); From 22905cf056136038955048bd00a9208be0e3e39d Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 01:01:24 -0400 Subject: [PATCH 026/175] closer --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 0cabb25..ffc5e47 100644 --- a/routes/api.js +++ b/routes/api.js @@ -65,6 +65,7 @@ var startWorkers = function(clworker, stopPercent){ stopPercent = stopPercent || 81; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent, clworker){ + console.log(arguments) if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ From 998063268f80cccec3e7d4c6591d90884c6891ee Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 01:03:44 -0400 Subject: [PATCH 027/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index ffc5e47..71d3ef6 100644 --- a/routes/api.js +++ b/routes/api.js @@ -64,7 +64,7 @@ var runner = function(req, res, container){ var startWorkers = function(clworker, stopPercent){ stopPercent = stopPercent || 81; console.log(clworker) - getFreeMem(clworker.ip, function(usedMemPercent, clworker){ + getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) if(usedMemPercent < 81 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From 30525c984c778c4cc28d80c2ad31f5408f02508b Mon Sep 17 00:00:00 2001 From: william Date: Thu, 5 May 2016 01:06:24 -0400 Subject: [PATCH 028/175] closer --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 71d3ef6..a23aac4 100644 --- a/routes/api.js +++ b/routes/api.js @@ -76,7 +76,7 @@ var startWorkers = function(clworker, stopPercent){ worker: clworker, label: clworker.name+':'+name }); - return startWorkers(clworker); + return setTimeout(startWorkers(clworker),0); }); }else{ console.log('using', usedMemPercent, 'percent memory, stopping container creation!', availContainers.length, 'created'); From 3b356a48f8c628806590a656917000c46f92919e Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 16:48:56 -0400 Subject: [PATCH 029/175] info --- routes/api.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/routes/api.js b/routes/api.js index a23aac4..b470509 100644 --- a/routes/api.js +++ b/routes/api.js @@ -10,7 +10,7 @@ var timeoutEvents = {}; var label2container = {}; var availContainers = []; var usedContainers = []; -var workers = { +var workers = { clworker0: { ip: '104.236.77.157', name: 'clworker0' @@ -28,7 +28,7 @@ var getFreeMem = function(ip, callback){ var lxcTimeout = function(container, time){ time = time || 900000; // 15 minutes - var keys = Object.keys(timeoutEvents) + var keys = Object.keys(timeoutEvents); if(keys.indexOf(container.label) !== -1){ clearTimeout(timeoutEvents[container.label]) @@ -38,8 +38,7 @@ var lxcTimeout = function(container, time){ lxc.stop(name, container); return startAll(container.worker); }, time); -} - +}; var runner = function(req, res, container){ lxcTimeout(container); @@ -51,10 +50,11 @@ var runner = function(req, res, container){ }, body: JSON.stringify({ code: req.body.code - }) + }); }; return request.post(httpOptions, function(error, response, body){ + console.log('runner:', arguments) body = JSON.parse(body); body['ip'] = container.label; return res.json(body); From 405569a0c97e05e23592c1bb3b1b305374cd8090 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 16:50:04 -0400 Subject: [PATCH 030/175] info --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index b470509..f659f64 100644 --- a/routes/api.js +++ b/routes/api.js @@ -62,11 +62,11 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 81; + stopPercent = stopPercent || 75; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) - if(usedMemPercent < 81 ){ + if(usedMemPercent < 75 ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ console.log('worker:', clworker.name, 'name:', name) From f5ea0966dc0e8ffa93eb2a1662f3e3c85c68f1ce Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 16:50:55 -0400 Subject: [PATCH 031/175] info --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index f659f64..a02b8fd 100644 --- a/routes/api.js +++ b/routes/api.js @@ -52,9 +52,9 @@ var runner = function(req, res, container){ code: req.body.code }); }; - + console.log('runner request:', httpOptions); return request.post(httpOptions, function(error, response, body){ - console.log('runner:', arguments) + console.log('runner response:', arguments) body = JSON.parse(body); body['ip'] = container.label; return res.json(body); From d00b3256254d0d2edbed44ccd344bf005be49e1b Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 16:52:18 -0400 Subject: [PATCH 032/175] info --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index a02b8fd..dc91818 100644 --- a/routes/api.js +++ b/routes/api.js @@ -50,7 +50,7 @@ var runner = function(req, res, container){ }, body: JSON.stringify({ code: req.body.code - }); + }) }; console.log('runner request:', httpOptions); return request.post(httpOptions, function(error, response, body){ From e6c45352c93c23fdbd9b22c4b576011b9231a4e5 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:01:52 -0400 Subject: [PATCH 033/175] info --- routes/api.js | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/routes/api.js b/routes/api.js index dc91818..df3d879 100644 --- a/routes/api.js +++ b/routes/api.js @@ -154,20 +154,18 @@ router.get('/list', function(req, res, next) { router.post('/run/:ip?', function doRun(req, res, next){ // check if server is + console.log('hit runner!') - return lxc.list(function(data){ - if(!req.params.ip) data = []; - var container = label2container[req.params.ip] || null; + var container = label2container[req.params.ip] || null; - if(container){ - return runner(req, res, container); - }else{ - container = availContainers.splice(0,1); - label2container[container.worker.name+':'+container.name] = container; - return runner(req, res, container); - } - }); + if(container){ + return runner(req, res, container); + }else{ + container = availContainers.splice(0,1); + label2container[container.worker.name+':'+container.name] = container; + return runner(req, res, container); + } }); From 152b206f9c8c0e4953cead515add37a620c068ff Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:02:36 -0400 Subject: [PATCH 034/175] info --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index df3d879..5ab943b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,6 +42,7 @@ var lxcTimeout = function(container, time){ var runner = function(req, res, container){ lxcTimeout(container); + console.log('calling runner:', container); var httpOptions = { url: 'http://' + container.worker.ip, From ae2a4984e486501fd4a0301e0fac4173dd5ff742 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:07:02 -0400 Subject: [PATCH 035/175] info --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 5ab943b..4db3a64 100644 --- a/routes/api.js +++ b/routes/api.js @@ -63,11 +63,11 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 75; + stopPercent = stopPercent || 10; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) - if(usedMemPercent < 75 ){ + if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ console.log('worker:', clworker.name, 'name:', name) From dd759efd8a9b6e93ca9b7a08879c38b886a2a195 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:17:23 -0400 Subject: [PATCH 036/175] info --- routes/api.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routes/api.js b/routes/api.js index 4db3a64..1ef1483 100644 --- a/routes/api.js +++ b/routes/api.js @@ -71,6 +71,8 @@ var startWorkers = function(clworker, stopPercent){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ console.log('worker:', clworker.name, 'name:', name) + if( !data.ip ) return setTimeout(startWorkers(clworker),0); + availContainers.push({ ip: data.ip, name: name, From 4257644d4e965060d71880642c52781b3a980057 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:18:04 -0400 Subject: [PATCH 037/175] info --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 1ef1483..90bb774 100644 --- a/routes/api.js +++ b/routes/api.js @@ -63,7 +63,7 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 10; + stopPercent = stopPercent || 20; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) From b8ce1fdcfcc50acbc30b0d355dca73281f706515 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:23:02 -0400 Subject: [PATCH 038/175] info --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 90bb774..9ea36ce 100644 --- a/routes/api.js +++ b/routes/api.js @@ -166,6 +166,7 @@ router.post('/run/:ip?', function doRun(req, res, next){ return runner(req, res, container); }else{ container = availContainers.splice(0,1); + console.log(container) label2container[container.worker.name+':'+container.name] = container; return runner(req, res, container); } From 4ca49a963251ca27e086e2973d4b7b77f5785b47 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:24:17 -0400 Subject: [PATCH 039/175] info --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 9ea36ce..d9fa8ea 100644 --- a/routes/api.js +++ b/routes/api.js @@ -63,7 +63,7 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 20; + stopPercent = stopPercent || 30; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) From d23f1b91d112be8150aef637de308120861082b8 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:26:56 -0400 Subject: [PATCH 040/175] info --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index d9fa8ea..813c552 100644 --- a/routes/api.js +++ b/routes/api.js @@ -63,7 +63,7 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 30; + stopPercent = stopPercent || 40; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) @@ -166,7 +166,7 @@ router.post('/run/:ip?', function doRun(req, res, next){ return runner(req, res, container); }else{ container = availContainers.splice(0,1); - console.log(container) + console.log(container[0]) label2container[container.worker.name+':'+container.name] = container; return runner(req, res, container); } From d4214cbf1d5a90793d54ca80296f0169269880d2 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:30:30 -0400 Subject: [PATCH 041/175] info --- routes/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 813c552..890ccb6 100644 --- a/routes/api.js +++ b/routes/api.js @@ -63,7 +63,7 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 40; + stopPercent = stopPercent || 20; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) @@ -165,8 +165,8 @@ router.post('/run/:ip?', function doRun(req, res, next){ if(container){ return runner(req, res, container); }else{ - container = availContainers.splice(0,1); - console.log(container[0]) + container = availContainers.splice(0,1)[0]; + console.log(container) label2container[container.worker.name+':'+container.name] = container; return runner(req, res, container); } From 360f2c591d1f36d58bfe759bbe559722ebca697b Mon Sep 17 00:00:00 2001 From: william Date: Mon, 16 May 2016 17:36:58 -0400 Subject: [PATCH 042/175] info --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 890ccb6..ebf740e 100644 --- a/routes/api.js +++ b/routes/api.js @@ -47,7 +47,7 @@ var runner = function(req, res, container){ var httpOptions = { url: 'http://' + container.worker.ip, headers: { - host: container.name + Host: container.name }, body: JSON.stringify({ code: req.body.code @@ -63,7 +63,7 @@ var runner = function(req, res, container){ }; var startWorkers = function(clworker, stopPercent){ - stopPercent = stopPercent || 20; + stopPercent = stopPercent || 30; console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ console.log(arguments) From 63b7a3b4812f836d737c0f6dca3f4ca490250c44 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 20 May 2016 09:11:31 -0400 Subject: [PATCH 043/175] do api started --- doapi.js | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/doapi.js b/doapi.js index 1cc5421..4d2abc6 100644 --- a/doapi.js +++ b/doapi.js @@ -2,50 +2,29 @@ var request = require('request'); api = function(key){ - if(!key){ - key = require('./secrets.js').doAPI; - } - this.url = 'https://digitalocean.com/v2/'; + key = key || require('./secrets.js').doAPI; + this.BASEURL = 'https://api.digitalocean.com/v2/'; + this.headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer '+key } - this.byTag: function(tag, callback){ + this.byTag = function(tag, callback){ var options = { - url: this.url+'droplets?tag_name='+tag, + url: this.BASEURL+'droplets?tag_name='+tag, headers: this.headers } + return request.get(options, function(error, response, body){ return callback(body, response, error); }) }; - this.setTag: function(id, tag, callback){ - return request.post(url+) - }; + // this.setTag = function(id, tag, callback){ + // return request.post(url+) + // }; + return this; } -api = { -} - - - - - - - -var httpOptions = { - url:'http://' + ip + ':15000', - body: JSON.stringify({ - code: req.body.code - }) - }; - - return request.post(httpOptions, function(error, response, body){ - body = JSON.parse(body); - body['ip'] = ip.replace('10.0.', ''); - return res.json(body); - }); - module.exports = api; \ No newline at end of file From 15cac181c38dad405a884b651817fd9b94da9967 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 23 May 2016 18:18:34 -0400 Subject: [PATCH 044/175] digital ocean API --- doapi.js | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 7 deletions(-) diff --git a/doapi.js b/doapi.js index 4d2abc6..9e417fd 100644 --- a/doapi.js +++ b/doapi.js @@ -10,21 +10,94 @@ api = function(key){ 'Authorization': 'Bearer '+key } - this.byTag = function(tag, callback){ + // API mathods + this.account = function(callback){ + var options = { + url: this.BASEURL+"account", + headers: this.headers + }; + + return request.get(options, function(error, response, body){ + return callback(body, response, error); + }); + }; + + this.dropletsByTag = function(tag, callback){ var options = { url: this.BASEURL+'droplets?tag_name='+tag, headers: this.headers - } + }; return request.get(options, function(error, response, body){ return callback(body, response, error); - }) + }); + }; + + this.dropletSetTag = function(tag, dropletID, callback) { + var data = { + resources: [ + { + resource_id: dropletID, + resource_type: "droplet" + } + ] + }; + var options = { + url: this.BASEURL+'tags/'+tag+'/resources', + headers: this.headers, + body: JSON.stringify(data) + }; + + return request.post(options, function(error, response, body){ + return callback(body, response, error); + }); + }; + + this.dropletCreate = function(args, callback){ + var data = { + name: args.name, // || return false, + region: args.region || "nyc3", + size: args.size || "512mb", + image: args.image || "ubuntu-14-04-x64", + ssh_keys: args.ssh_key || null, + backups: args.backup || false, + private_networking: args.private_networking || true, + user_data: args.user_data || null + }; + var options = { + url: this.BASEURL+'droplets', + headers: this.headers, + body: JSON.stringify(data) + }; + + return request.post(options, function(error, response, body){ + return callback(body, response, error); + }); + } + + this.dropletDestroy = function(dropletID, callback){ + var options = { + url: this.BASEURL+"droplets/"+dropletID, + headers: this.headers + }; + + return request.del(options, function(error, response, body){ + callback(body, response, error); + }); + }; + + this.dropletInfo = function(dropletID, callback){ + var options = { + url: this.BASEURL+"droplets/"+dropletID, + headers: this.headers + }; + + return request.get(options, function(error, response, body){ + callback(body, response, error); + }); }; - // this.setTag = function(id, tag, callback){ - // return request.post(url+) - // }; return this; } -module.exports = api; \ No newline at end of file +module.exports = api; From 32e70d4be467401cbdf37861ea7771f9486fd696 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 23 May 2016 18:25:38 -0400 Subject: [PATCH 045/175] digital ocean API --- doapi.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/doapi.js b/doapi.js index 9e417fd..4ebcf0f 100644 --- a/doapi.js +++ b/doapi.js @@ -1,19 +1,16 @@ var request = require('request'); - api = function(key){ key = key || require('./secrets.js').doAPI; this.BASEURL = 'https://api.digitalocean.com/v2/'; - this.headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer '+key } - // API mathods this.account = function(callback){ var options = { - url: this.BASEURL+"account", + url: this.BASEURL+'account', headers: this.headers }; @@ -38,7 +35,7 @@ api = function(key){ resources: [ { resource_id: dropletID, - resource_type: "droplet" + resource_type: 'droplet' } ] }; @@ -56,9 +53,9 @@ api = function(key){ this.dropletCreate = function(args, callback){ var data = { name: args.name, // || return false, - region: args.region || "nyc3", - size: args.size || "512mb", - image: args.image || "ubuntu-14-04-x64", + region: args.region || 'nyc3', + size: args.size || '512mb', + image: args.image || 'ubuntu-14-04-x64', ssh_keys: args.ssh_key || null, backups: args.backup || false, private_networking: args.private_networking || true, @@ -77,7 +74,7 @@ api = function(key){ this.dropletDestroy = function(dropletID, callback){ var options = { - url: this.BASEURL+"droplets/"+dropletID, + url: this.BASEURL+'droplets/'+dropletID, headers: this.headers }; @@ -88,7 +85,7 @@ api = function(key){ this.dropletInfo = function(dropletID, callback){ var options = { - url: this.BASEURL+"droplets/"+dropletID, + url: this.BASEURL+'droplets/'+dropletID, headers: this.headers }; From f503f370b2d0626699b217667533b457242bdc29 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 15:38:58 -0400 Subject: [PATCH 046/175] stuff --- routes/api.js | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/routes/api.js b/routes/api.js index ebf740e..ae78346 100644 --- a/routes/api.js +++ b/routes/api.js @@ -5,17 +5,20 @@ var router = express.Router(); var extend = require('node.extend'); var request = require('request'); var lxc = require('../lxc'); +var doapi = require('../doapi')(); var timeoutEvents = {}; var label2container = {}; var availContainers = []; var usedContainers = []; -var workers = { - clworker0: { - ip: '104.236.77.157', - name: 'clworker0' - } -}; +var workers = {}; + +// var workers = { +// clworker0: { +// ip: '104.236.77.157', +// name: 'clworker0' +// } +// }; var getFreeMem = function(ip, callback){ @@ -26,6 +29,15 @@ var getFreeMem = function(ip, callback){ ); }; +var getWorkers = function(){ + doapi.dropletsByTag('clworker', function(data){ + data = JSON.parse(data); + data.forEach(function(value){ + workers[value.name] = makeWokerObj(value); + }); + }); +}; + var lxcTimeout = function(container, time){ time = time || 900000; // 15 minutes var keys = Object.keys(timeoutEvents); @@ -62,6 +74,14 @@ var runner = function(req, res, container){ }); }; +var makeWokerObj = function(woker){ + worker.networks.forEach(function(value){ + worker[value.type+'IP'] = value.ip_address; + }); + worker.ip = worker.privateIP; + return worker; +}; + var startWorkers = function(clworker, stopPercent){ stopPercent = stopPercent || 30; console.log(clworker) From f875bf101a0f409cde1ed23ed895ab72823f1d92 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 16:04:36 -0400 Subject: [PATCH 047/175] workers --- routes/api.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index ae78346..7f66ca6 100644 --- a/routes/api.js +++ b/routes/api.js @@ -34,6 +34,7 @@ var getWorkers = function(){ data = JSON.parse(data); data.forEach(function(value){ workers[value.name] = makeWokerObj(value); + startWorkers(workers[value.name]); }); }); }; @@ -107,7 +108,8 @@ var startWorkers = function(clworker, stopPercent){ }); }; -startWorkers(workers.clworker0); + +getWorkers(); router.get('/start/:name', function(req, res, next){ return lxc.start(req.params.name, function(data){ @@ -163,6 +165,10 @@ router.get('/destroy/:name', function(req, res, next){ }); }); +router.get('/liststuff', function(req, res, next){ + res.json({'workers': workers, 'availContainers': availContainers}) +}); + router.get('/info/:name', function(req, res, next){ return lxc.info(req.params.name, function(data){ return res.json(data); From a1338c5792291b954532f15951148cb3ba5ad5ea Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 16:06:16 -0400 Subject: [PATCH 048/175] workers --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 7f66ca6..493eba0 100644 --- a/routes/api.js +++ b/routes/api.js @@ -32,7 +32,7 @@ var getFreeMem = function(ip, callback){ var getWorkers = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); - data.forEach(function(value){ + data['droplets'].forEach(function(value){ workers[value.name] = makeWokerObj(value); startWorkers(workers[value.name]); }); From b53c3e68088f716b244915e7e146553f56a2e554 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 16:07:01 -0400 Subject: [PATCH 049/175] workers --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 493eba0..ad6a2ef 100644 --- a/routes/api.js +++ b/routes/api.js @@ -75,7 +75,7 @@ var runner = function(req, res, container){ }); }; -var makeWokerObj = function(woker){ +var makeWokerObj = function(worker){ worker.networks.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); From 16f91f01a770d243ec143749c5ea6cce49d14f22 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 16:07:43 -0400 Subject: [PATCH 050/175] workers --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index ad6a2ef..54d394c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -76,6 +76,7 @@ var runner = function(req, res, container){ }; var makeWokerObj = function(worker){ + console.log(worker); worker.networks.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); From baa8fe673081d2f4c97e9c68983a064332dab3e0 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 16:08:36 -0400 Subject: [PATCH 051/175] workers --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 54d394c..4a4d0de 100644 --- a/routes/api.js +++ b/routes/api.js @@ -77,7 +77,7 @@ var runner = function(req, res, container){ var makeWokerObj = function(worker){ console.log(worker); - worker.networks.forEach(function(value){ + worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); worker.ip = worker.privateIP; From eff29a9922bc6cd95320f66f0ab44e8abe28ac03 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 25 May 2016 23:28:22 -0400 Subject: [PATCH 052/175] stuff --- routes/api.js | 1 - 1 file changed, 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 4a4d0de..36281b8 100644 --- a/routes/api.js +++ b/routes/api.js @@ -76,7 +76,6 @@ var runner = function(req, res, container){ }; var makeWokerObj = function(worker){ - console.log(worker); worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); From 4651a2f49c13803999686904aabd250b4078ce8a Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 19:22:06 -0400 Subject: [PATCH 053/175] stuff --- routes/api.js | 207 ++++++++++++++++++++++++++------------------------ 1 file changed, 108 insertions(+), 99 deletions(-) diff --git a/routes/api.js b/routes/api.js index 36281b8..f3bef27 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,18 +7,12 @@ var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); -var timeoutEvents = {}; var label2container = {}; -var availContainers = []; -var usedContainers = []; -var workers = {}; +var workers = []; -// var workers = { -// clworker0: { -// ip: '104.236.77.157', -// name: 'clworker0' -// } -// }; +var checkBalance = function(){ + +}; var getFreeMem = function(ip, callback){ @@ -29,32 +23,26 @@ var getFreeMem = function(ip, callback){ ); }; -var getWorkers = function(){ - doapi.dropletsByTag('clworker', function(data){ - data = JSON.parse(data); - data['droplets'].forEach(function(value){ - workers[value.name] = makeWokerObj(value); - startWorkers(workers[value.name]); - }); - }); +var containerFree = function(container){ + lxc.stop(name, container); + container.worker.usedContainer--; + delete label2container[container.label]; }; var lxcTimeout = function(container, time){ time = time || 900000; // 15 minutes - var keys = Object.keys(timeoutEvents); - if(keys.indexOf(container.label) !== -1){ - clearTimeout(timeoutEvents[container.label]) + if(container.hasOwnProperty('timeout')){ + clearTimeout(container.timeout); } - return timeoutEvents[container.label] = setTimeout(function(){ - lxc.stop(name, container); + return container.timeout = setTimeout(function(){ + containerFree(container) return startAll(container.worker); }, time); }; var runner = function(req, res, container){ - lxcTimeout(container); console.log('calling runner:', container); var httpOptions = { @@ -66,12 +54,21 @@ var runner = function(req, res, container){ code: req.body.code }) }; - console.log('runner request:', httpOptions); + return request.post(httpOptions, function(error, response, body){ console.log('runner response:', arguments) body = JSON.parse(body); + var i = -1; + while(workers[++i].availContainers.length && workers[i].index < container.worker.index){ + containerFree(container); + + container = workers[i].availContainers.pop(); + container.usedContainer++; + } body['ip'] = container.label; + lxcTimeout(container); return res.json(body); + }); }; @@ -79,58 +76,113 @@ var makeWokerObj = function(worker){ worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); + worker.availContainers = []; worker.ip = worker.privateIP; + worker.usedContainer = 0; return worker; }; -var startWorkers = function(clworker, stopPercent){ +var getWorkers = function(){ + doapi.dropletsByTag('clworker', function(data){ + data = JSON.parse(data); + data['droplets'].forEach(function(value){ + var workerIDX = workers.push(makeWokerObj(value)) - 1; + workers[workerIDX].index = workerIDX; + + startWorker(workers[workerIDX]); + }); + }); +}; + +var getAvailContainer = function(){ + var i = -1; + while(workers[++i].availContainers.length){ + var container = wrokers[i].availContainers.pop(); + label2container[container.label] = container; + container.worker.usedContainer++; + return container; + } +}; + +var startWorker = function(clworker, stopPercent){ stopPercent = stopPercent || 30; - console.log(clworker) getFreeMem(clworker.ip, function(usedMemPercent){ - console.log(arguments) if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ - console.log('worker:', clworker.name, 'name:', name) - if( !data.ip ) return setTimeout(startWorkers(clworker),0); - availContainers.push({ + if( !data.ip ) return setTimeout(startWorker(clworker),0); + + worker.availContainers.push({ ip: data.ip, name: name, worker: clworker, label: clworker.name+':'+name }); - return setTimeout(startWorkers(clworker),0); + return setTimeout(startWorker(clworker, stopPercent), 0); }); }else{ - console.log('using', usedMemPercent, 'percent memory, stopping container creation!', availContainers.length, 'created'); + console.log('using', usedMemPercent, 'percent memory, stopping container creation!', worker.availContainers.length, 'created'); } }); }; - getWorkers(); -router.get('/start/:name', function(req, res, next){ - return lxc.start(req.params.name, function(data){ - if(!data){ - return res.json({ - status: 500, - name: req.params.name, - message: data - }); - }else{ - res.json({}); - } - }); -}); +// router.get('/start/:name', function(req, res, next){ +// return lxc.start(req.params.name, function(data){ +// if(!data){ +// return res.json({ +// status: 500, +// name: req.params.name, +// message: data +// }); +// }else{ +// res.json({}); +// } +// }); +// }); -router.get('/live/:template/:name', function(req, res, next){ - return lxc.startEphemeral(req.params.name, req.params.template, function (data) { - console.log('live', arguments); - return res.json(data); - }); -}); +// router.get('/live/:template/:name', function(req, res, next){ +// return lxc.startEphemeral(req.params.name, req.params.template, function (data) { +// console.log('live', arguments); +// return res.json(data); +// }); +// }); + +// router.get('/clone/:template/:name', function(req, res, next){ +// return lxc.clone(req.params.name, req.params.template, function(data){ +// console.log('clone', arguments); +// if( data.match(/Created container/) ){ +// return res.json({status: 200}); +// }else{ +// return res.json({status: 500, message: data}); +// } +// }); +// }); + +// router.get('/destroy/:name', function(req, res, next){ +// return lxc.destroy(req.params.name, function(data){ +// console.log('destroy', arguments); +// if(data){ +// return res.json({status: 500, message: data}); +// }else{ +// return res.json({status: 200}); +// } +// }); +// }); + +// router.get('/info/:name', function(req, res, next){ +// return lxc.info(req.params.name, function(data){ +// return res.json(data); +// }); +// }); + +// router.get('/list', function(req, res, next) { +// return lxc.list(workers.clworker0.ip, function(data){ +// return res.json(data); +// }); +// }); router.get('/stop/:name', function(req, res, next){ return lxc.stop(req.params.name, function(data){ @@ -143,60 +195,17 @@ router.get('/stop/:name', function(req, res, next){ }); }); -router.get('/clone/:template/:name', function(req, res, next){ - return lxc.clone(req.params.name, req.params.template, function(data){ - console.log('clone', arguments); - if( data.match(/Created container/) ){ - return res.json({status: 200}); - }else{ - return res.json({status: 500, message: data}); - } - }); -}); - -router.get('/destroy/:name', function(req, res, next){ - return lxc.destroy(req.params.name, function(data){ - console.log('destroy', arguments); - if(data){ - return res.json({status: 500, message: data}); - }else{ - return res.json({status: 200}); - } - }); -}); - router.get('/liststuff', function(req, res, next){ res.json({'workers': workers, 'availContainers': availContainers}) }); -router.get('/info/:name', function(req, res, next){ - return lxc.info(req.params.name, function(data){ - return res.json(data); - }); -}); - -router.get('/list', function(req, res, next) { - return lxc.list(workers.clworker0.ip, function(data){ - return res.json(data); - }); -}); - router.post('/run/:ip?', function doRun(req, res, next){ // check if server is console.log('hit runner!') - var container = label2container[req.params.ip] || null; - - - if(container){ - return runner(req, res, container); - }else{ - container = availContainers.splice(0,1)[0]; - console.log(container) - label2container[container.worker.name+':'+container.name] = container; - return runner(req, res, container); - } - + var container = label2container[req.params.ip] || getAvailContainer(); + + return runner(req, res, container); }); module.exports = router; From b74a71efb4bd708549feb3a05c4bbab9562b5baf Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 19:23:22 -0400 Subject: [PATCH 054/175] stuff --- routes/api.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/routes/api.js b/routes/api.js index f3bef27..eb5d73c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -104,22 +104,22 @@ var getAvailContainer = function(){ } }; -var startWorker = function(clworker, stopPercent){ +var startWorker = function(worker, stopPercent){ stopPercent = stopPercent || 30; - getFreeMem(clworker.ip, function(usedMemPercent){ + getFreeMem(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); - return lxc.startEphemeral(name, 'crunner0', clworker.ip, function(data){ + return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if( !data.ip ) return setTimeout(startWorker(clworker),0); + if( !data.ip ) return setTimeout(startWorker(worker),0); worker.availContainers.push({ ip: data.ip, name: name, - worker: clworker, - label: clworker.name+':'+name + worker: worker, + label: worker.name+':'+name }); - return setTimeout(startWorker(clworker, stopPercent), 0); + return setTimeout(startWorker(worker, stopPercent), 0); }); }else{ console.log('using', usedMemPercent, 'percent memory, stopping container creation!', worker.availContainers.length, 'created'); From 0e7ccf2637db145dfc47772a81045f95fb46baff Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 19:24:07 -0400 Subject: [PATCH 055/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index eb5d73c..492312d 100644 --- a/routes/api.js +++ b/routes/api.js @@ -110,8 +110,8 @@ var startWorker = function(worker, stopPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if( !data.ip ) return setTimeout(startWorker(worker),0); + console.log('started container') worker.availContainers.push({ ip: data.ip, From 61dedb013d07d67ce7c90ce5aba37575719b7c3d Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 19:47:13 -0400 Subject: [PATCH 056/175] stuff --- routes/api.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 492312d..50b1a9e 100644 --- a/routes/api.js +++ b/routes/api.js @@ -97,6 +97,7 @@ var getWorkers = function(){ var getAvailContainer = function(){ var i = -1; while(workers[++i].availContainers.length){ + console.log('found avail'); var container = wrokers[i].availContainers.pop(); label2container[container.label] = container; container.worker.usedContainer++; @@ -117,7 +118,7 @@ var startWorker = function(worker, stopPercent){ ip: data.ip, name: name, worker: worker, - label: worker.name+':'+name + label: worker.name + ':' + name }); return setTimeout(startWorker(worker, stopPercent), 0); }); @@ -196,7 +197,7 @@ router.get('/stop/:name', function(req, res, next){ }); router.get('/liststuff', function(req, res, next){ - res.json({'workers': workers, 'availContainers': availContainers}) + res.json({'workers': workers}) }); router.post('/run/:ip?', function doRun(req, res, next){ From ed2b37d696be5f53b59adb467c0fd795997652ac Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 19:52:16 -0400 Subject: [PATCH 057/175] stuff --- routes/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 50b1a9e..080380d 100644 --- a/routes/api.js +++ b/routes/api.js @@ -97,10 +97,11 @@ var getWorkers = function(){ var getAvailContainer = function(){ var i = -1; while(workers[++i].availContainers.length){ - console.log('found avail'); + console.log('found avail on worker:', i); var container = wrokers[i].availContainers.pop(); label2container[container.label] = container; container.worker.usedContainer++; + console.log('return container:', container); return container; } }; From 66f6c3b92acc73a369287fe37e14ac0812faeeba Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 19:54:24 -0400 Subject: [PATCH 058/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 080380d..79be5ac 100644 --- a/routes/api.js +++ b/routes/api.js @@ -98,7 +98,7 @@ var getAvailContainer = function(){ var i = -1; while(workers[++i].availContainers.length){ console.log('found avail on worker:', i); - var container = wrokers[i].availContainers.pop(); + var container = workers[i].availContainers.pop(); label2container[container.label] = container; container.worker.usedContainer++; console.log('return container:', container); @@ -107,7 +107,7 @@ var getAvailContainer = function(){ }; var startWorker = function(worker, stopPercent){ - stopPercent = stopPercent || 30; + stopPercent = stopPercent || 60; getFreeMem(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From f32d6f70a95697981336106c6b1b1f21cd3e2a88 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 20:14:01 -0400 Subject: [PATCH 059/175] stuff --- lxc.js | 8 ++++---- routes/api.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lxc.js b/lxc.js index 96d4bec..b8e1472 100644 --- a/lxc.js +++ b/lxc.js @@ -30,7 +30,7 @@ var lxc = { destroy: function(name, callback){ return sysExec('lxc-destroy -n '+ name, function(data){ var info = data.match(/Destroyed container/); - console.log('destroy info:', info); + // console.log('destroy info:', info); var args = [true].concat(Array.prototype.slice.call(arguments, 1)); return callback.apply(this, args); }); @@ -43,7 +43,7 @@ var lxc = { startEphemeral: function(name, base_name, ip, callback){ var command = 'lxc-start-ephemeral -o '+base_name+ ' -n '+name +' --union-type overlayfs -d'; return sysExec(command, ip, function(data){ - console.log('startEphemeral', arguments); + // console.log('startEphemeral', arguments); if(data.match("doesn't exist.")){ return callback({status: 500, error: "doesn't exist."}); } @@ -58,7 +58,7 @@ var lxc = { }); }, - stop: function(name, callback, ip){ + stop: function(name, ip, callback){ return sysExec('lxc-stop -n '+ name, ip, callback); }, @@ -72,7 +72,7 @@ var lxc = { info: function(name, callback){ return sysExec('lxc-info -n '+name, function(data){ - console.log('info', arguments); + // console.log('info', arguments); if(data.match("doesn't exist")){ return callback({state: 'NULL'}); } diff --git a/routes/api.js b/routes/api.js index 79be5ac..75bf84a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -24,7 +24,7 @@ var getFreeMem = function(ip, callback){ }; var containerFree = function(container){ - lxc.stop(name, container); + lxc.stop(container.name, container.worker.ip); container.worker.usedContainer--; delete label2container[container.label]; }; From 19293fdaff27f9c7f4c26a28b173530e756fd098 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 20:15:58 -0400 Subject: [PATCH 060/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 75bf84a..9d6ee6b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -107,7 +107,7 @@ var getAvailContainer = function(){ }; var startWorker = function(worker, stopPercent){ - stopPercent = stopPercent || 60; + stopPercent = stopPercent || 80; getFreeMem(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From caf32c3809bece1d8314c64cfcb49506ec0f1d7f Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 20:41:25 -0400 Subject: [PATCH 061/175] stuff --- routes/api.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/routes/api.js b/routes/api.js index 9d6ee6b..656b035 100644 --- a/routes/api.js +++ b/routes/api.js @@ -2,7 +2,7 @@ var express = require('express'); var router = express.Router(); -var extend = require('node.extend'); +var util = require('util'); var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); @@ -43,7 +43,6 @@ var lxcTimeout = function(container, time){ }; var runner = function(req, res, container){ - console.log('calling runner:', container); var httpOptions = { url: 'http://' + container.worker.ip, @@ -56,7 +55,7 @@ var runner = function(req, res, container){ }; return request.post(httpOptions, function(error, response, body){ - console.log('runner response:', arguments) + // console.log('runner response:', arguments) body = JSON.parse(body); var i = -1; while(workers[++i].availContainers.length && workers[i].index < container.worker.index){ @@ -97,11 +96,11 @@ var getWorkers = function(){ var getAvailContainer = function(){ var i = -1; while(workers[++i].availContainers.length){ - console.log('found avail on worker:', i); + var container = workers[i].availContainers.pop(); label2container[container.label] = container; container.worker.usedContainer++; - console.log('return container:', container); + return container; } }; @@ -198,7 +197,9 @@ router.get('/stop/:name', function(req, res, next){ }); router.get('/liststuff', function(req, res, next){ - res.json({'workers': workers}) + var obj = util.inspect(workers); + console.log('worker inspected:', obj) + res.send(obj); }); router.post('/run/:ip?', function doRun(req, res, next){ From c6e9ec62ba25a1c1f24fa1fc2ea865d7b1440329 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 20:58:17 -0400 Subject: [PATCH 062/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 656b035..87a5626 100644 --- a/routes/api.js +++ b/routes/api.js @@ -27,6 +27,7 @@ var containerFree = function(container){ lxc.stop(container.name, container.worker.ip); container.worker.usedContainer--; delete label2container[container.label]; + return startWorker(container.worker); }; var lxcTimeout = function(container, time){ @@ -38,7 +39,6 @@ var lxcTimeout = function(container, time){ return container.timeout = setTimeout(function(){ containerFree(container) - return startAll(container.worker); }, time); }; From 129cd54812c57ccdde8934057a936cffd1cb1dba Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 21:16:02 -0400 Subject: [PATCH 063/175] stuff --- routes/api.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 87a5626..ca3f33f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -31,7 +31,7 @@ var containerFree = function(container){ }; var lxcTimeout = function(container, time){ - time = time || 900000; // 15 minutes + time = time || 60000 // 900000; // 15 minutes if(container.hasOwnProperty('timeout')){ clearTimeout(container.timeout); @@ -197,8 +197,7 @@ router.get('/stop/:name', function(req, res, next){ }); router.get('/liststuff', function(req, res, next){ - var obj = util.inspect(workers); - console.log('worker inspected:', obj) + var obj = util.inspect(workers, {depth: 4}); res.send(obj); }); From 5fc47fbbcc11cc8ead365a494130da126a72dfe8 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 26 May 2016 21:21:15 -0400 Subject: [PATCH 064/175] stuff --- routes/api.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index ca3f33f..09a6c72 100644 --- a/routes/api.js +++ b/routes/api.js @@ -106,6 +106,7 @@ var getAvailContainer = function(){ }; var startWorker = function(worker, stopPercent){ + console.log('starting runners on', worker.name) stopPercent = stopPercent || 80; getFreeMem(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ @@ -202,8 +203,6 @@ router.get('/liststuff', function(req, res, next){ }); router.post('/run/:ip?', function doRun(req, res, next){ - // check if server is - console.log('hit runner!') var container = label2container[req.params.ip] || getAvailContainer(); From 1af8aed161b88d46ab6cbe8566fb4e71f150091d Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:32:03 -0400 Subject: [PATCH 065/175] stuff --- routes/api.js | 154 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 103 insertions(+), 51 deletions(-) diff --git a/routes/api.js b/routes/api.js index 09a6c72..0124816 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,14 +7,63 @@ var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); -var label2container = {}; +var label2runner = {}; var workers = []; +var isCheckingWorkers = false; -var checkBalance = function(){ - +var checkWorker = function(id, time){ + time = time || 30000; + doapi.dropletInfo(id, function(data){ + worker = JSON.parse(data)['droplet']; + if(worker.status == 'active'){ + workers.push(makeWokerObj(worker)); + isCheckingWorkers = false; + return ch; + }else if(worker.status == 'new'){ + setTimeout(function(){ + checkWorker(id) + }, time); + } + }); }; -var getFreeMem = function(ip, callback){ +var workerCreate = function(){ + doapi.dropletCreate({ + name: 'clworker'+(Math.random()*100).toString().replace('.',''), + image: '17375637' + }, function(data){ + data = JSON.parse(data); + doapi.dropletSetTag('clworker', data.droplet.id, function(data){ + setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, time);) + }); + }); +}; + +var workerDestroy = function(worker){ + worker = worker || workers.pop(); + doapi.dropletDestroy(worker.id, function(){}); +}; + +var checkWorkers = function(){ + if(isCheckingWorkers) return false; + + isCheckingWorkers = true; + if(!workers ){ + return workerCreate(); + } + if(workers[workers.length-1].usedrunner){ + return workerCreate(); + } + if(workers[workers.length-1].usedrunner && workers[workers.length-2].usedrunner){ + + workerDestroy(); + } + isCheckingWorkers = false; +}; + +var start + +var ramPercentUsed = function(ip, callback){ return lxc.exec( "python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"", @@ -23,31 +72,35 @@ var getFreeMem = function(ip, callback){ ); }; -var containerFree = function(container){ - lxc.stop(container.name, container.worker.ip); - container.worker.usedContainer--; - delete label2container[container.label]; - return startWorker(container.worker); +var runnerFree = function(runner){ + lxc.stop(runner.name, runner.worker.ip); + runner.worker.usedrunner--; + if(runner.hasOwnProperty('timeout')){ + clearTimeout(runner.timeout); + } + delete label2runner[runner.label]; + + startRunners(runner.worker); }; -var lxcTimeout = function(container, time){ +var lxcTimeout = function(runner, time){ time = time || 60000 // 900000; // 15 minutes - if(container.hasOwnProperty('timeout')){ - clearTimeout(container.timeout); + if(runner.hasOwnProperty('timeout')){ + clearTimeout(runner.timeout); } - return container.timeout = setTimeout(function(){ - containerFree(container) + return runner.timeout = setTimeout(function(){ + runnerFree(runner) }, time); }; -var runner = function(req, res, container){ +var runner = function(req, res, runner){ var httpOptions = { - url: 'http://' + container.worker.ip, + url: 'http://' + runner.worker.ip, headers: { - Host: container.name + Host: runner.name }, body: JSON.stringify({ code: req.body.code @@ -57,15 +110,9 @@ var runner = function(req, res, container){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) body = JSON.parse(body); - var i = -1; - while(workers[++i].availContainers.length && workers[i].index < container.worker.index){ - containerFree(container); - container = workers[i].availContainers.pop(); - container.usedContainer++; - } - body['ip'] = container.label; - lxcTimeout(container); + body['ip'] = getAvailrunner(runner).label; + lxcTimeout(runner); return res.json(body); }); @@ -75,61 +122,66 @@ var makeWokerObj = function(worker){ worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); - worker.availContainers = []; + worker.availrunners = []; worker.ip = worker.privateIP; - worker.usedContainer = 0; + worker.usedrunner = 0; + worker.index = workers.length, + worker.getRunner = function(){ + if(this.availrunners === 0) return false; + var runner = this.availrunners.pop(); + this.usedrunner++; + label2runner[runner.label] = runner; + + return runner; + } return worker; }; -var getWorkers = function(){ +var initWorkers = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); data['droplets'].forEach(function(value){ - var workerIDX = workers.push(makeWokerObj(value)) - 1; - workers[workerIDX].index = workerIDX; - - startWorker(workers[workerIDX]); + startRunners(workers[workers.push(makeWokerObj(value))-1]); }); }); }; -var getAvailContainer = function(){ +var getAvailrunner = function(runner){ var i = -1; - while(workers[++i].availContainers.length){ - - var container = workers[i].availContainers.pop(); - label2container[container.label] = container; - container.worker.usedContainer++; - - return container; + while(workers[++i].availrunners.length && ( runner && workers[i].index < runner.worker.index )){ + if(runner) runnerFree(runner); + return workers[i].getRunner(); } + if(runner) return runner; + }; -var startWorker = function(worker, stopPercent){ +var startRunners = function(worker, stopPercent){ console.log('starting runners on', worker.name) stopPercent = stopPercent || 80; - getFreeMem(worker.ip, function(usedMemPercent){ + ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if( !data.ip ) return setTimeout(startWorker(worker),0); - console.log('started container') + if( !data.ip ) return setTimeout(startRunners(worker),0); + console.log('started runner') - worker.availContainers.push({ + worker.availrunners.push({ ip: data.ip, name: name, worker: worker, label: worker.name + ':' + name }); - return setTimeout(startWorker(worker, stopPercent), 0); + return setTimeout(startRunners(worker, stopPercent), 0); }); }else{ - console.log('using', usedMemPercent, 'percent memory, stopping container creation!', worker.availContainers.length, 'created'); + checkWorker(); + console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created'); } }); }; -getWorkers(); +initWorkers(); // router.get('/start/:name', function(req, res, next){ // return lxc.start(req.params.name, function(data){ @@ -155,7 +207,7 @@ getWorkers(); // router.get('/clone/:template/:name', function(req, res, next){ // return lxc.clone(req.params.name, req.params.template, function(data){ // console.log('clone', arguments); -// if( data.match(/Created container/) ){ +// if( data.match(/Created runner/) ){ // return res.json({status: 200}); // }else{ // return res.json({status: 500, message: data}); @@ -204,9 +256,9 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ - var container = label2container[req.params.ip] || getAvailContainer(); + var runner = label2runner[req.params.ip] || getAvailrunner(); - return runner(req, res, container); + return runner(req, res, runner); }); module.exports = router; From 7d4965b21e722d4c7f5165380f3b2c79983b91b2 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:34:24 -0400 Subject: [PATCH 066/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 0124816..e6c933e 100644 --- a/routes/api.js +++ b/routes/api.js @@ -158,7 +158,7 @@ var getAvailrunner = function(runner){ var startRunners = function(worker, stopPercent){ console.log('starting runners on', worker.name) - stopPercent = stopPercent || 80; + stopPercent = stopPercent || 15; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From b8f23077dafe941af2973756b1618f4af85942ea Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:34:52 -0400 Subject: [PATCH 067/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index e6c933e..24c334e 100644 --- a/routes/api.js +++ b/routes/api.js @@ -34,7 +34,7 @@ var workerCreate = function(){ }, function(data){ data = JSON.parse(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, time);) + setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, time)) }); }); }; From 26f06c0aa8f58ffa481d6438117bdd461a55d5c9 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:35:57 -0400 Subject: [PATCH 068/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 24c334e..7123e35 100644 --- a/routes/api.js +++ b/routes/api.js @@ -14,7 +14,7 @@ var isCheckingWorkers = false; var checkWorker = function(id, time){ time = time || 30000; doapi.dropletInfo(id, function(data){ - worker = JSON.parse(data)['droplet']; + var worker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ workers.push(makeWokerObj(worker)); isCheckingWorkers = false; From 9a0ad761b9048fc8eda9aea4c3749db85b96fcfd Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:37:54 -0400 Subject: [PATCH 069/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 7123e35..a652ef5 100644 --- a/routes/api.js +++ b/routes/api.js @@ -33,6 +33,7 @@ var workerCreate = function(){ image: '17375637' }, function(data){ data = JSON.parse(data); + console.log(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, time)) }); From 1ddda799d312ee4d22aa550d213223ba7ff3dc66 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:38:52 -0400 Subject: [PATCH 070/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index a652ef5..73031eb 100644 --- a/routes/api.js +++ b/routes/api.js @@ -35,7 +35,7 @@ var workerCreate = function(){ data = JSON.parse(data); console.log(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, time)) + setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, 10000)) }); }); }; From 62fcae727a4dfac58eb937a44e712ef51ec349fe Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:40:59 -0400 Subject: [PATCH 071/175] stuff --- routes/api.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/routes/api.js b/routes/api.js index 73031eb..7c72615 100644 --- a/routes/api.js +++ b/routes/api.js @@ -11,7 +11,7 @@ var label2runner = {}; var workers = []; var isCheckingWorkers = false; -var checkWorker = function(id, time){ +var checkDroplet = function(id, time){ time = time || 30000; doapi.dropletInfo(id, function(data){ var worker = JSON.parse(data)['droplet']; @@ -21,7 +21,7 @@ var checkWorker = function(id, time){ return ch; }else if(worker.status == 'new'){ setTimeout(function(){ - checkWorker(id) + checkDroplet(id) }, time); } }); @@ -35,7 +35,7 @@ var workerCreate = function(){ data = JSON.parse(data); console.log(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(setTimeout(function(){checkWorker(data.droplet.id)}, 10000)) + setTimeout(setTimeout(function(){checkDroplet(data.droplet.id)}, 10000)) }); }); }; @@ -45,7 +45,7 @@ var workerDestroy = function(worker){ doapi.dropletDestroy(worker.id, function(){}); }; -var checkWorkers = function(){ +var checkWorkersBalance = function(){ if(isCheckingWorkers) return false; isCheckingWorkers = true; @@ -176,7 +176,7 @@ var startRunners = function(worker, stopPercent){ return setTimeout(startRunners(worker, stopPercent), 0); }); }else{ - checkWorker(); + checkWorkersBalance(); console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created'); } }); From d4da910040a48c4c7c2e447f9efada8b363d929c Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:43:07 -0400 Subject: [PATCH 072/175] stuff --- routes/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7c72615..9dca5ae 100644 --- a/routes/api.js +++ b/routes/api.js @@ -49,13 +49,13 @@ var checkWorkersBalance = function(){ if(isCheckingWorkers) return false; isCheckingWorkers = true; - if(!workers ){ + if(!workers){ return workerCreate(); } - if(workers[workers.length-1].usedrunner){ + if(!workers[workers.length-1].availrunners.length){ return workerCreate(); } - if(workers[workers.length-1].usedrunner && workers[workers.length-2].usedrunner){ + if(!workers[workers.length-1].availrunners.length && !workers[workers.length-2].availrunners.length){ workerDestroy(); } From a301ad88e566e58b00bee06bc10ef800cbb13275 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:45:38 -0400 Subject: [PATCH 073/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 9dca5ae..514e5a2 100644 --- a/routes/api.js +++ b/routes/api.js @@ -35,7 +35,7 @@ var workerCreate = function(){ data = JSON.parse(data); console.log(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(setTimeout(function(){checkDroplet(data.droplet.id)}, 10000)) + setTimeout(function(){checkDroplet(data.droplet.id)}, 10000); }); }); }; From c31948714185954d16b729fcfcc1afa29bf7127b Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:50:04 -0400 Subject: [PATCH 074/175] stuff --- routes/api.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 514e5a2..0a0cbdf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -11,6 +11,8 @@ var label2runner = {}; var workers = []; var isCheckingWorkers = false; +var dopletNewID = 0; + var checkDroplet = function(id, time){ time = time || 30000; doapi.dropletInfo(id, function(data){ @@ -33,9 +35,10 @@ var workerCreate = function(){ image: '17375637' }, function(data){ data = JSON.parse(data); - console.log(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(function(){checkDroplet(data.droplet.id)}, 10000); + console.log('Set tag data:', data) + dopletNewID = data.droplet.id; + setTimeout(function(){checkDroplet(dropletNewId)}, 10000); }); }); }; From 8c32b2e2468a2de64183619288d3294d2b336a5a Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:52:07 -0400 Subject: [PATCH 075/175] stuff --- lxc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lxc.js b/lxc.js index b8e1472..1b6acf0 100644 --- a/lxc.js +++ b/lxc.js @@ -4,7 +4,7 @@ var exec = require('child_process').exec; function sysExec(command, ip, callback){ ip = ip || '104.236.77.157'; command = new Buffer(command).toString('base64') - command = 'ssh virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; + command = 'ssh -o UserKnownHostsFile=/dev/null virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ From 1086baf65aff941f6b37851a06b9d9963dc2ec29 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:52:39 -0400 Subject: [PATCH 076/175] stuff --- lxc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lxc.js b/lxc.js index 1b6acf0..9bf3796 100644 --- a/lxc.js +++ b/lxc.js @@ -4,7 +4,7 @@ var exec = require('child_process').exec; function sysExec(command, ip, callback){ ip = ip || '104.236.77.157'; command = new Buffer(command).toString('base64') - command = 'ssh -o UserKnownHostsFile=/dev/null virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; + command = 'ssh -o StrictHostKeyChecking=no virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"'; // command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command; return exec(command, (function(callback){ From 28596a0b4d1f54a6d5dc6d04659a6a1544f6cf49 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:53:19 -0400 Subject: [PATCH 077/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 0a0cbdf..8b5ff3d 100644 --- a/routes/api.js +++ b/routes/api.js @@ -35,9 +35,9 @@ var workerCreate = function(){ image: '17375637' }, function(data){ data = JSON.parse(data); + dopletNewID = data.droplet.id; doapi.dropletSetTag('clworker', data.droplet.id, function(data){ console.log('Set tag data:', data) - dopletNewID = data.droplet.id; setTimeout(function(){checkDroplet(dropletNewId)}, 10000); }); }); From 9a58acdfa207772530cd39c47b439cf74f45687d Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:56:37 -0400 Subject: [PATCH 078/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 8b5ff3d..bfc31a4 100644 --- a/routes/api.js +++ b/routes/api.js @@ -179,7 +179,7 @@ var startRunners = function(worker, stopPercent){ return setTimeout(startRunners(worker, stopPercent), 0); }); }else{ - checkWorkersBalance(); + setTimeout(checkWorkersBalance, 30000); console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created'); } }); From 7ba58ee8d68e2ae8ca0061e31afd8a6ea9543b67 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:58:10 -0400 Subject: [PATCH 079/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index bfc31a4..d309b13 100644 --- a/routes/api.js +++ b/routes/api.js @@ -38,7 +38,7 @@ var workerCreate = function(){ dopletNewID = data.droplet.id; doapi.dropletSetTag('clworker', data.droplet.id, function(data){ console.log('Set tag data:', data) - setTimeout(function(){checkDroplet(dropletNewId)}, 10000); + setTimeout(function(){checkDroplet(dopletNewID)}, 10000); }); }); }; @@ -180,7 +180,7 @@ var startRunners = function(worker, stopPercent){ }); }else{ setTimeout(checkWorkersBalance, 30000); - console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created'); + console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); } }); }; From d7b32996b3448e0d73ab58de83cd7dad4144ef61 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 00:59:28 -0400 Subject: [PATCH 080/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index d309b13..f10cce9 100644 --- a/routes/api.js +++ b/routes/api.js @@ -21,7 +21,7 @@ var checkDroplet = function(id, time){ workers.push(makeWokerObj(worker)); isCheckingWorkers = false; return ch; - }else if(worker.status == 'new'){ + }else{ setTimeout(function(){ checkDroplet(id) }, time); From ac524a1c769319c5678643464f782c0dd3618ab9 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 01:05:15 -0400 Subject: [PATCH 081/175] stuff --- routes/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index f10cce9..175428e 100644 --- a/routes/api.js +++ b/routes/api.js @@ -14,13 +14,13 @@ var isCheckingWorkers = false; var dopletNewID = 0; var checkDroplet = function(id, time){ - time = time || 30000; + time = time || 5000; doapi.dropletInfo(id, function(data){ var worker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ workers.push(makeWokerObj(worker)); isCheckingWorkers = false; - return ch; + return true; }else{ setTimeout(function(){ checkDroplet(id) @@ -56,6 +56,7 @@ var checkWorkersBalance = function(){ return workerCreate(); } if(!workers[workers.length-1].availrunners.length){ + console.log('starting new droplet!'); return workerCreate(); } if(!workers[workers.length-1].availrunners.length && !workers[workers.length-2].availrunners.length){ @@ -179,7 +180,7 @@ var startRunners = function(worker, stopPercent){ return setTimeout(startRunners(worker, stopPercent), 0); }); }else{ - setTimeout(checkWorkersBalance, 30000); + setTimeout(checkWorkersBalance, 10000); console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); } }); From ee3091ae3c821f2af2eff4e4142499b57870a93c Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 10:45:42 -0400 Subject: [PATCH 082/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 175428e..4f44fa2 100644 --- a/routes/api.js +++ b/routes/api.js @@ -55,11 +55,11 @@ var checkWorkersBalance = function(){ if(!workers){ return workerCreate(); } - if(!workers[workers.length-1].availrunners.length){ + if(workers[workers.length-1].availrunners.length === 0){ console.log('starting new droplet!'); return workerCreate(); } - if(!workers[workers.length-1].availrunners.length && !workers[workers.length-2].availrunners.length){ + if(workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ workerDestroy(); } From 99043948db2502dde524da99d00c7da4eeb9582d Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 10:52:46 -0400 Subject: [PATCH 083/175] stuff --- routes/api.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 4f44fa2..ab8ab88 100644 --- a/routes/api.js +++ b/routes/api.js @@ -18,7 +18,7 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ var worker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ - workers.push(makeWokerObj(worker)); + startRunners(workers[workers.push(makeWokerObj(worker))-1]); isCheckingWorkers = false; return true; }else{ @@ -37,7 +37,6 @@ var workerCreate = function(){ data = JSON.parse(data); dopletNewID = data.droplet.id; doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - console.log('Set tag data:', data) setTimeout(function(){checkDroplet(dopletNewID)}, 10000); }); }); From bf21a8b9666d8e71dfd2ce820cd88f0e6cdb863a Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 11:07:20 -0400 Subject: [PATCH 084/175] stuff --- routes/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index ab8ab88..364544a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -18,7 +18,7 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ var worker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ - startRunners(workers[workers.push(makeWokerObj(worker))-1]); + setTimeout(startRunners(workers[workers.push(makeWokerObj(worker))-1]), 5000); isCheckingWorkers = false; return true; }else{ @@ -52,14 +52,15 @@ var checkWorkersBalance = function(){ isCheckingWorkers = true; if(!workers){ + console.log('No workers, starting droplet'); return workerCreate(); } if(workers[workers.length-1].availrunners.length === 0){ - console.log('starting new droplet!'); + console.log('last droplet has no free runners, starting droplet'); return workerCreate(); } if(workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ - + console.log('Last 2 runners not used, killing last runner'); workerDestroy(); } isCheckingWorkers = false; From dd68a292086acd5aaa7c7814bc2570b47e03604e Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 11:11:38 -0400 Subject: [PATCH 085/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 364544a..ba1f1ac 100644 --- a/routes/api.js +++ b/routes/api.js @@ -51,7 +51,7 @@ var checkWorkersBalance = function(){ if(isCheckingWorkers) return false; isCheckingWorkers = true; - if(!workers){ + if(workers.length < 2){ console.log('No workers, starting droplet'); return workerCreate(); } From 5a0ab51c321838a678050fa519880c8b0f435c56 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 13:07:44 -0400 Subject: [PATCH 086/175] stuff --- routes/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index ba1f1ac..ccedea7 100644 --- a/routes/api.js +++ b/routes/api.js @@ -12,13 +12,14 @@ var workers = []; var isCheckingWorkers = false; var dopletNewID = 0; +var newWorker = {}; var checkDroplet = function(id, time){ time = time || 5000; doapi.dropletInfo(id, function(data){ - var worker = JSON.parse(data)['droplet']; + newWorker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ - setTimeout(startRunners(workers[workers.push(makeWokerObj(worker))-1]), 5000); + setTimeout(startRunners(function(){workers[workers.push(makeWokerObj(newWorker))-1]}), 5000); isCheckingWorkers = false; return true; }else{ @@ -59,7 +60,7 @@ var checkWorkersBalance = function(){ console.log('last droplet has no free runners, starting droplet'); return workerCreate(); } - if(workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + if(workers.length>1 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner'); workerDestroy(); } From a4f01f80625c5e0ec0a2a93b9ac4dbc2c8492918 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 13:10:45 -0400 Subject: [PATCH 087/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index ccedea7..1ad2202 100644 --- a/routes/api.js +++ b/routes/api.js @@ -18,7 +18,7 @@ var checkDroplet = function(id, time){ time = time || 5000; doapi.dropletInfo(id, function(data){ newWorker = JSON.parse(data)['droplet']; - if(worker.status == 'active'){ + if(newWorker.status == 'active'){ setTimeout(startRunners(function(){workers[workers.push(makeWokerObj(newWorker))-1]}), 5000); isCheckingWorkers = false; return true; From 6b72ddcdd4ff15dc58240e0865e80dd8af369454 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 13:14:44 -0400 Subject: [PATCH 088/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 1ad2202..3577917 100644 --- a/routes/api.js +++ b/routes/api.js @@ -19,7 +19,7 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ newWorker = JSON.parse(data)['droplet']; if(newWorker.status == 'active'){ - setTimeout(startRunners(function(){workers[workers.push(makeWokerObj(newWorker))-1]}), 5000); + setTimeout(function(){startRunners(workers[workers.push(makeWokerObj(newWorker))-1])}, 5000); isCheckingWorkers = false; return true; }else{ From 7fcbcecb6a1fe0398be28e466d4cf667b6219840 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 15:02:32 -0400 Subject: [PATCH 089/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 3577917..aefe528 100644 --- a/routes/api.js +++ b/routes/api.js @@ -164,7 +164,7 @@ var getAvailrunner = function(runner){ var startRunners = function(worker, stopPercent){ console.log('starting runners on', worker.name) - stopPercent = stopPercent || 15; + stopPercent = stopPercent || 30; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From 9991e7eb3658931426c8a34c1b3ed96381fbc0fb Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 15:28:28 -0400 Subject: [PATCH 090/175] stuff --- routes/api.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/routes/api.js b/routes/api.js index aefe528..465fb6a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -19,7 +19,9 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ newWorker = JSON.parse(data)['droplet']; if(newWorker.status == 'active'){ - setTimeout(function(){startRunners(workers[workers.push(makeWokerObj(newWorker))-1])}, 5000); + setTimeout(function(){ + startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) + }, 5000); isCheckingWorkers = false; return true; }else{ @@ -46,6 +48,7 @@ var workerCreate = function(){ var workerDestroy = function(worker){ worker = worker || workers.pop(); doapi.dropletDestroy(worker.id, function(){}); + checkWorkersBalance(); }; var checkWorkersBalance = function(){ @@ -60,7 +63,7 @@ var checkWorkersBalance = function(){ console.log('last droplet has no free runners, starting droplet'); return workerCreate(); } - if(workers.length>1 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner'); workerDestroy(); } @@ -97,7 +100,7 @@ var lxcTimeout = function(runner, time){ } return runner.timeout = setTimeout(function(){ - runnerFree(runner) + runnerFree(runner); }, time); }; @@ -124,7 +127,7 @@ var runner = function(req, res, runner){ }); }; -var makeWokerObj = function(worker){ +var makeWorkerObj = function(worker){ worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); @@ -147,14 +150,14 @@ var initWorkers = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); data['droplets'].forEach(function(value){ - startRunners(workers[workers.push(makeWokerObj(value))-1]); + startRunners(workers[workers.push(makeWorkerObj(value))-1]); }); }); }; var getAvailrunner = function(runner){ var i = -1; - while(workers[++i].availrunners.length && ( runner && workers[i].index < runner.worker.index )){ + while(workers[++i].availrunners.length !== 0 /*&& ( runner && workers[i].index < runner.worker.index )*/){ if(runner) runnerFree(runner); return workers[i].getRunner(); } From 8a5001aee70833ab84ab6c74251f5c8ece12cb8f Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 15:39:41 -0400 Subject: [PATCH 091/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 465fb6a..68bebd0 100644 --- a/routes/api.js +++ b/routes/api.js @@ -59,7 +59,7 @@ var checkWorkersBalance = function(){ console.log('No workers, starting droplet'); return workerCreate(); } - if(workers[workers.length-1].availrunners.length === 0){ + if(workers[workers.length-1].usedrunner !== 0){ console.log('last droplet has no free runners, starting droplet'); return workerCreate(); } From 371a40a3d3a80a1626584dc8ae7fbacddf4f39ff Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 15:45:19 -0400 Subject: [PATCH 092/175] stuff --- routes/api.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/routes/api.js b/routes/api.js index 68bebd0..488d60f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -21,7 +21,7 @@ var checkDroplet = function(id, time){ if(newWorker.status == 'active'){ setTimeout(function(){ startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) - }, 5000); + }, 10000); isCheckingWorkers = false; return true; }else{ @@ -156,12 +156,12 @@ var initWorkers = function(){ }; var getAvailrunner = function(runner){ - var i = -1; - while(workers[++i].availrunners.length !== 0 /*&& ( runner && workers[i].index < runner.worker.index )*/){ - if(runner) runnerFree(runner); - return workers[i].getRunner(); + for(let worker of workers){ + if(!worker.availrunners) continue; + // if(runner) runnerFree(runner); + return worker.getRunner(); } - if(runner) return runner; + // if(runner) return runner; }; From 101160e8875e0215d9a97d7039e6e8347569ecc4 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 15:54:07 -0400 Subject: [PATCH 093/175] stuff --- routes/api.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 488d60f..252f549 100644 --- a/routes/api.js +++ b/routes/api.js @@ -137,6 +137,7 @@ var makeWorkerObj = function(worker){ worker.index = workers.length, worker.getRunner = function(){ if(this.availrunners === 0) return false; + console.log('geting runner from ', worker.name, ' aval length ', this.availrunners); var runner = this.availrunners.pop(); this.usedrunner++; label2runner[runner.label] = runner; @@ -264,9 +265,9 @@ router.get('/liststuff', function(req, res, next){ }); router.post('/run/:ip?', function doRun(req, res, next){ - + console.log('hit runner route') var runner = label2runner[req.params.ip] || getAvailrunner(); - + console.log('') return runner(req, res, runner); }); From 71bd4ffc4a1f696e5891e59c215e96487ee69350 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 19:35:30 -0400 Subject: [PATCH 094/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 252f549..e4170d8 100644 --- a/routes/api.js +++ b/routes/api.js @@ -158,7 +158,7 @@ var initWorkers = function(){ var getAvailrunner = function(runner){ for(let worker of workers){ - if(!worker.availrunners) continue; + if(worker.availrunners.length !== 0) continue; // if(runner) runnerFree(runner); return worker.getRunner(); } From d5bd5d2c39062f33faeeaf1f930ff4c2fc880df5 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 19:47:34 -0400 Subject: [PATCH 095/175] stuff --- routes/api.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index e4170d8..8c23276 100644 --- a/routes/api.js +++ b/routes/api.js @@ -19,12 +19,14 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ newWorker = JSON.parse(data)['droplet']; if(newWorker.status == 'active'){ + console.log('Runner is now active, starting runners in 10 seconds') setTimeout(function(){ startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) }, 10000); isCheckingWorkers = false; return true; }else{ + console.log('Worker not ready, check again in ', time, 'MS'); setTimeout(function(){ checkDroplet(id) }, time); @@ -56,7 +58,7 @@ var checkWorkersBalance = function(){ isCheckingWorkers = true; if(workers.length < 2){ - console.log('No workers, starting droplet'); + console.log('less then 2 workers, starting a droplet'); return workerCreate(); } if(workers[workers.length-1].usedrunner !== 0){ @@ -67,6 +69,7 @@ var checkWorkersBalance = function(){ console.log('Last 2 runners not used, killing last runner'); workerDestroy(); } + console.log('stopping workers balancing check'); isCheckingWorkers = false; }; @@ -158,7 +161,8 @@ var initWorkers = function(){ var getAvailrunner = function(runner){ for(let worker of workers){ - if(worker.availrunners.length !== 0) continue; + console.log('checking ', worker.name, ' with ', worker.availrunners.length, ' free workers'); + if(worker.availrunners.length === 0) continue; // if(runner) runnerFree(runner); return worker.getRunner(); } From eb9b8b25565832883e238bbbf29646750d79f84c Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 19:48:04 -0400 Subject: [PATCH 096/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 8c23276..044b4e8 100644 --- a/routes/api.js +++ b/routes/api.js @@ -21,6 +21,7 @@ var checkDroplet = function(id, time){ if(newWorker.status == 'active'){ console.log('Runner is now active, starting runners in 10 seconds') setTimeout(function(){ + console.log('Ready to start runners!') startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) }, 10000); isCheckingWorkers = false; From f6f5b81b495256420acee6f51ba5a334b30f6359 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 19:53:46 -0400 Subject: [PATCH 097/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 044b4e8..9bd62d7 100644 --- a/routes/api.js +++ b/routes/api.js @@ -140,8 +140,8 @@ var makeWorkerObj = function(worker){ worker.usedrunner = 0; worker.index = workers.length, worker.getRunner = function(){ - if(this.availrunners === 0) return false; - console.log('geting runner from ', worker.name, ' aval length ', this.availrunners); + if(this.availrunners.length === 0) return false; + console.log('geting runner from ', worker.name, ' aval length ', this.availrunners.length); var runner = this.availrunners.pop(); this.usedrunner++; label2runner[runner.label] = runner; From 424b5c68c8e87fd1eb5f75f6be36d9b2a0f16b4b Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 19:54:47 -0400 Subject: [PATCH 098/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 9bd62d7..682f82c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -19,11 +19,11 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ newWorker = JSON.parse(data)['droplet']; if(newWorker.status == 'active'){ - console.log('Runner is now active, starting runners in 10 seconds') + console.log('Runner is now active, starting runners in 15 seconds') setTimeout(function(){ console.log('Ready to start runners!') startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) - }, 10000); + }, 15000); isCheckingWorkers = false; return true; }else{ From 732d15cc54650d33141b95339a9ef677240fcf54 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 20:02:27 -0400 Subject: [PATCH 099/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 682f82c..c85622a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -108,7 +108,7 @@ var lxcTimeout = function(runner, time){ }, time); }; -var runner = function(req, res, runner){ +var run = function(req, res, runner){ var httpOptions = { url: 'http://' + runner.worker.ip, @@ -273,7 +273,7 @@ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route') var runner = label2runner[req.params.ip] || getAvailrunner(); console.log('') - return runner(req, res, runner); + return run(req, res, runner); }); module.exports = router; From b5737a0470bece0755b5b50ecc15415aedba6fe1 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 20:07:32 -0400 Subject: [PATCH 100/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index c85622a..8d9be64 100644 --- a/routes/api.js +++ b/routes/api.js @@ -173,7 +173,7 @@ var getAvailrunner = function(runner){ var startRunners = function(worker, stopPercent){ console.log('starting runners on', worker.name) - stopPercent = stopPercent || 30; + stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); From 18897cd4e693e88f63564ddd40e38653021b1795 Mon Sep 17 00:00:00 2001 From: william Date: Fri, 27 May 2016 20:37:14 -0400 Subject: [PATCH 101/175] stuff --- routes/api.js | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/routes/api.js b/routes/api.js index 8d9be64..ee43662 100644 --- a/routes/api.js +++ b/routes/api.js @@ -19,12 +19,12 @@ var checkDroplet = function(id, time){ doapi.dropletInfo(id, function(data){ newWorker = JSON.parse(data)['droplet']; if(newWorker.status == 'active'){ - console.log('Runner is now active, starting runners in 15 seconds') + console.log('Droplet is now active, starting runners in 20 seconds') setTimeout(function(){ console.log('Ready to start runners!') startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) - }, 15000); - isCheckingWorkers = false; + isCheckingWorkers = false; + }, 20000); return true; }else{ console.log('Worker not ready, check again in ', time, 'MS'); @@ -43,7 +43,7 @@ var workerCreate = function(){ data = JSON.parse(data); dopletNewID = data.droplet.id; doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(function(){checkDroplet(dopletNewID)}, 10000); + setTimeout(function(){checkDroplet(dopletNewID)}, 60000); }); }); }; @@ -56,7 +56,7 @@ var workerDestroy = function(worker){ var checkWorkersBalance = function(){ if(isCheckingWorkers) return false; - + var changed = false; isCheckingWorkers = true; if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); @@ -70,8 +70,17 @@ var checkWorkersBalance = function(){ console.log('Last 2 runners not used, killing last runner'); workerDestroy(); } + + for(let worker of workers){ + if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + workerDestroy(worker); + changed = true; + } + } + console.log('stopping workers balancing check'); isCheckingWorkers = false; + if(changed) checkWorkersBalance(); }; var start @@ -164,11 +173,12 @@ var getAvailrunner = function(runner){ for(let worker of workers){ console.log('checking ', worker.name, ' with ', worker.availrunners.length, ' free workers'); if(worker.availrunners.length === 0) continue; - // if(runner) runnerFree(runner); + if(runner && runner.worker.index <= worker.index) break; + if(runner) runnerFree(runner); return worker.getRunner(); } - // if(runner) return runner; - + if(runner) return runner; + return false; }; var startRunners = function(worker, stopPercent){ From 3e592887d1bf1d8d42418e5e2e640e6affe88765 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 00:13:24 -0400 Subject: [PATCH 102/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index ee43662..8a88c70 100644 --- a/routes/api.js +++ b/routes/api.js @@ -38,7 +38,7 @@ var checkDroplet = function(id, time){ var workerCreate = function(){ doapi.dropletCreate({ name: 'clworker'+(Math.random()*100).toString().replace('.',''), - image: '17375637' + image: '17575764' }, function(data){ data = JSON.parse(data); dopletNewID = data.droplet.id; From b5baacf668393c19e12e47ce2c69e488a831af71 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 00:18:10 -0400 Subject: [PATCH 103/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 8a88c70..c1bcf02 100644 --- a/routes/api.js +++ b/routes/api.js @@ -163,6 +163,7 @@ var makeWorkerObj = function(worker){ var initWorkers = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); + if(data.droplets.length === 0) return checkWorkersBalance(); data['droplets'].forEach(function(value){ startRunners(workers[workers.push(makeWorkerObj(value))-1]); }); From 191851e2bbb8254cbafe7498eedfa2ccfbb43fa3 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:08:17 -0400 Subject: [PATCH 104/175] stuff --- routes/api.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index c1bcf02..882b2f0 100644 --- a/routes/api.js +++ b/routes/api.js @@ -80,7 +80,9 @@ var checkWorkersBalance = function(){ console.log('stopping workers balancing check'); isCheckingWorkers = false; - if(changed) checkWorkersBalance(); + if(changed) setTimeout(function(){ + checkWorkersBalance(); + }, 3000); }; var start @@ -131,6 +133,7 @@ var run = function(req, res, runner){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) + console.log() body = JSON.parse(body); body['ip'] = getAvailrunner(runner).label; From ccd128100af47fa00dfb29fa761fa960196b748c Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:40:01 -0400 Subject: [PATCH 105/175] stuff --- routes/api.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/routes/api.js b/routes/api.js index 882b2f0..c1b76bf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,6 +7,10 @@ var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); +var workerTag = clworker; +var workerShapID = '17575764' + + var label2runner = {}; var workers = []; var isCheckingWorkers = false; @@ -37,12 +41,12 @@ var checkDroplet = function(id, time){ var workerCreate = function(){ doapi.dropletCreate({ - name: 'clworker'+(Math.random()*100).toString().replace('.',''), + name: 'clw'+workerShapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' }, function(data){ data = JSON.parse(data); dopletNewID = data.droplet.id; - doapi.dropletSetTag('clworker', data.droplet.id, function(data){ + doapi.dropletSetTag('clw'+'17575764', data.droplet.id, function(data){ setTimeout(function(){checkDroplet(dopletNewID)}, 60000); }); }); @@ -56,8 +60,8 @@ var workerDestroy = function(worker){ var checkWorkersBalance = function(){ if(isCheckingWorkers) return false; - var changed = false; isCheckingWorkers = true; + var changed = false; if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); return workerCreate(); @@ -133,7 +137,7 @@ var run = function(req, res, runner){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) - console.log() + if(error) return false; body = JSON.parse(body); body['ip'] = getAvailrunner(runner).label; @@ -164,11 +168,11 @@ var makeWorkerObj = function(worker){ }; var initWorkers = function(){ - doapi.dropletsByTag('clworker', function(data){ + doapi.dropletsByTag('clw'+workerShapID, function(data){ data = JSON.parse(data); if(data.droplets.length === 0) return checkWorkersBalance(); - data['droplets'].forEach(function(value){ - startRunners(workers[workers.push(makeWorkerObj(value))-1]); + data['droplets'].forEach(function(worker){ + doapi.dropletDestroy(worker.id, function(){}); }); }); }; From a51f4c203c642c17e693f53d8f174fcaa2845937 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:40:35 -0400 Subject: [PATCH 106/175] stuff --- routes/api.js | 1 - 1 file changed, 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index c1b76bf..5f838df 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,7 +7,6 @@ var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); -var workerTag = clworker; var workerShapID = '17575764' From f86a74a5892c81862e1925fe01c2cf6659305d2a Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:47:38 -0400 Subject: [PATCH 107/175] stuff --- routes/api.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 5f838df..481b0ff 100644 --- a/routes/api.js +++ b/routes/api.js @@ -9,6 +9,7 @@ var doapi = require('../doapi')(); var workerShapID = '17575764' +console.log = function(){}; var label2runner = {}; var workers = []; @@ -57,11 +58,11 @@ var workerDestroy = function(worker){ checkWorkersBalance(); }; -var checkWorkersBalance = function(){ +var checkWorkersBalance = function(count){ if(isCheckingWorkers) return false; isCheckingWorkers = true; var changed = false; - if(workers.length < 2){ + if(workers.length < count){ console.log('less then 2 workers, starting a droplet'); return workerCreate(); } From b84c87d55d10a36613d1d64af843a258892abf9f Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:48:40 -0400 Subject: [PATCH 108/175] stuff --- routes/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 481b0ff..cd3b3e2 100644 --- a/routes/api.js +++ b/routes/api.js @@ -9,7 +9,7 @@ var doapi = require('../doapi')(); var workerShapID = '17575764' -console.log = function(){}; +// console.log = function(){}; var label2runner = {}; var workers = []; @@ -62,6 +62,7 @@ var checkWorkersBalance = function(count){ if(isCheckingWorkers) return false; isCheckingWorkers = true; var changed = false; + if(workers.length < count){ console.log('less then 2 workers, starting a droplet'); return workerCreate(); From 2ba7fa58e282d9c9a423592120d02913b5432f92 Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:49:41 -0400 Subject: [PATCH 109/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index cd3b3e2..4af5d05 100644 --- a/routes/api.js +++ b/routes/api.js @@ -62,7 +62,7 @@ var checkWorkersBalance = function(count){ if(isCheckingWorkers) return false; isCheckingWorkers = true; var changed = false; - + console.log('checking balance'); if(workers.length < count){ console.log('less then 2 workers, starting a droplet'); return workerCreate(); From 586c8cc32fa5a6666a329b90eb251b3acb3d3e4b Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:54:17 -0400 Subject: [PATCH 110/175] stuff --- routes/api.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/routes/api.js b/routes/api.js index 4af5d05..8867970 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,7 +7,7 @@ var request = require('request'); var lxc = require('../lxc'); var doapi = require('../doapi')(); -var workerShapID = '17575764' +var workerSnapID = 'V1' // console.log = function(){}; @@ -41,12 +41,12 @@ var checkDroplet = function(id, time){ var workerCreate = function(){ doapi.dropletCreate({ - name: 'clw'+workerShapID+'-'+(Math.random()*100).toString().replace('.',''), + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' }, function(data){ data = JSON.parse(data); dopletNewID = data.droplet.id; - doapi.dropletSetTag('clw'+'17575764', data.droplet.id, function(data){ + doapi.dropletSetTag('clw'+workerSnapID, data.droplet.id, function(data){ setTimeout(function(){checkDroplet(dopletNewID)}, 60000); }); }); @@ -90,7 +90,7 @@ var checkWorkersBalance = function(count){ }, 3000); }; -var start +// var start var ramPercentUsed = function(ip, callback){ @@ -169,12 +169,13 @@ var makeWorkerObj = function(worker){ }; var initWorkers = function(){ - doapi.dropletsByTag('clw'+workerShapID, function(data){ + doapi.dropletsByTag('clw'+workerSnapID, function(data){ data = JSON.parse(data); - if(data.droplets.length === 0) return checkWorkersBalance(); data['droplets'].forEach(function(worker){ + doapi.dropletDestroy(worker.id, function(){}); }); + checkWorkersBalance(); }); }; From e6dee5bde367735af57b4b79e38157481fe55a8e Mon Sep 17 00:00:00 2001 From: william Date: Sat, 28 May 2016 01:55:21 -0400 Subject: [PATCH 111/175] stuff --- routes/api.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 8867970..f1e4968 100644 --- a/routes/api.js +++ b/routes/api.js @@ -58,12 +58,13 @@ var workerDestroy = function(worker){ checkWorkersBalance(); }; -var checkWorkersBalance = function(count){ +var checkWorkersBalance = function(){ if(isCheckingWorkers) return false; isCheckingWorkers = true; var changed = false; console.log('checking balance'); - if(workers.length < count){ + + if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); return workerCreate(); } From 0b9b5c062714ca2266ea84c3f7e9b1ff66c5cded Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 00:36:56 -0400 Subject: [PATCH 112/175] stuff --- routes/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index f1e4968..04b20b6 100644 --- a/routes/api.js +++ b/routes/api.js @@ -66,7 +66,8 @@ var checkWorkersBalance = function(){ if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); - return workerCreate(); + for(var i=0; i<2) workerCreate(); + return ; } if(workers[workers.length-1].usedrunner !== 0){ console.log('last droplet has no free runners, starting droplet'); From 02c0c854175111b787c414859844127b18ca497d Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 00:37:43 -0400 Subject: [PATCH 113/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 04b20b6..269ae41 100644 --- a/routes/api.js +++ b/routes/api.js @@ -66,7 +66,7 @@ var checkWorkersBalance = function(){ if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); - for(var i=0; i<2) workerCreate(); + for(var i=0; i<2; i++) workerCreate(); return ; } if(workers[workers.length-1].usedrunner !== 0){ From 571e81ace57f249db921c4d25a888b0eb8ce93cb Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 00:47:29 -0400 Subject: [PATCH 114/175] stuff --- routes/api.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/routes/api.js b/routes/api.js index 269ae41..8eccac9 100644 --- a/routes/api.js +++ b/routes/api.js @@ -39,17 +39,22 @@ var checkDroplet = function(id, time){ }); }; -var workerCreate = function(){ +var workerCreate = function(count){ doapi.dropletCreate({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' }, function(data){ data = JSON.parse(data); dopletNewID = data.droplet.id; - doapi.dropletSetTag('clw'+workerSnapID, data.droplet.id, function(data){ - setTimeout(function(){checkDroplet(dopletNewID)}, 60000); + doapi.dropletSetTag('clworker', data.droplet.id, function(data){ + setTimeout(function(){ + checkDroplet(dopletNewID) + }, 60000); }); }); + if(count) setTimeout(function(){ + workerCreate(--count); + }, 1000); }; var workerDestroy = function(worker){ @@ -171,7 +176,7 @@ var makeWorkerObj = function(worker){ }; var initWorkers = function(){ - doapi.dropletsByTag('clw'+workerSnapID, function(data){ + doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); data['droplets'].forEach(function(worker){ From a7e1478697422604c18945750011138e937b7dae Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 00:52:26 -0400 Subject: [PATCH 115/175] stuff --- routes/api.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 8eccac9..2434866 100644 --- a/routes/api.js +++ b/routes/api.js @@ -40,6 +40,7 @@ var checkDroplet = function(id, time){ }; var workerCreate = function(count){ + console.log('creating worker. ', count); doapi.dropletCreate({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' @@ -52,9 +53,10 @@ var workerCreate = function(count){ }, 60000); }); }); - if(count) setTimeout(function(){ + if(count) setTimeout(function(){ + console.log('making ', count-1, 'more'); workerCreate(--count); - }, 1000); + }, 2000); }; var workerDestroy = function(worker){ @@ -71,7 +73,7 @@ var checkWorkersBalance = function(){ if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); - for(var i=0; i<2; i++) workerCreate(); + workerCreate(2); return ; } if(workers[workers.length-1].usedrunner !== 0){ From 7d12414d74ac6cfef752e6837667e8c3e4048554 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 01:03:46 -0400 Subject: [PATCH 116/175] stuff --- routes/api.js | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/routes/api.js b/routes/api.js index 2434866..60f6e8a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -16,19 +16,18 @@ var workers = []; var isCheckingWorkers = false; var dopletNewID = 0; -var newWorker = {}; var checkDroplet = function(id, time){ time = time || 5000; doapi.dropletInfo(id, function(data){ - newWorker = JSON.parse(data)['droplet']; - if(newWorker.status == 'active'){ + var worker = JSON.parse(data)['droplet']; + if(worker.status == 'active'){ console.log('Droplet is now active, starting runners in 20 seconds') - setTimeout(function(){ + setTimeout(function(worker){ console.log('Ready to start runners!') - startRunners(workers[workers.push(makeWorkerObj(newWorker))-1]) + startRunners(workers[workers.push(makeWorkerObj(worker))-1]) isCheckingWorkers = false; - }, 20000); + }, 20000, worker); return true; }else{ console.log('Worker not ready, check again in ', time, 'MS'); @@ -39,24 +38,18 @@ var checkDroplet = function(id, time){ }); }; -var workerCreate = function(count){ - console.log('creating worker. ', count); +var workerCreate = function(){ doapi.dropletCreate({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' }, function(data){ - data = JSON.parse(data); - dopletNewID = data.droplet.id; doapi.dropletSetTag('clworker', data.droplet.id, function(data){ - setTimeout(function(){ + setTimeout(function(dopletNewID){ checkDroplet(dopletNewID) - }, 60000); + }, 60000, JSON.parse(data).droplet.id); }); }); - if(count) setTimeout(function(){ - console.log('making ', count-1, 'more'); - workerCreate(--count); - }, 2000); + }; var workerDestroy = function(worker){ @@ -73,7 +66,7 @@ var checkWorkersBalance = function(){ if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); - workerCreate(2); + for(var i=2; i--;) workerCreate(); return ; } if(workers[workers.length-1].usedrunner !== 0){ From cd37c680f073ae30e9f55d3314bda1efa0610598 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 01:05:54 -0400 Subject: [PATCH 117/175] stuff --- routes/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 60f6e8a..3f73a0a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -43,10 +43,11 @@ var workerCreate = function(){ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' }, function(data){ + data = JSON.parse(data); doapi.dropletSetTag('clworker', data.droplet.id, function(data){ setTimeout(function(dopletNewID){ checkDroplet(dopletNewID) - }, 60000, JSON.parse(data).droplet.id); + }, 60000, data.droplet.id); }); }); From 4079ee8208ee7c0dfe3dce27b5404d0ae34adf55 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 01:09:01 -0400 Subject: [PATCH 118/175] stuff --- routes/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 3f73a0a..1249bba 100644 --- a/routes/api.js +++ b/routes/api.js @@ -44,7 +44,8 @@ var workerCreate = function(){ image: '17575764' }, function(data){ data = JSON.parse(data); - doapi.dropletSetTag('clworker', data.droplet.id, function(data){ + doapi.dropletSetTag('clworker', data.droplet.id, function(d){ + console.log(d) setTimeout(function(dopletNewID){ checkDroplet(dopletNewID) }, 60000, data.droplet.id); From 42ea77720ac2b0d9d3aa54e9b5f99dfa385747ef Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 01:33:22 -0400 Subject: [PATCH 119/175] stuff --- doapi.js | 11 +++++++++++ routes/api.js | 8 ++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/doapi.js b/doapi.js index 4ebcf0f..0959a55 100644 --- a/doapi.js +++ b/doapi.js @@ -94,6 +94,17 @@ api = function(key){ }); }; + this.tagsList = function(callback){ + var options = { + url: this.BASEURL+'tags', + headers: this.headers + }; + + return request.get(options, function(e,r,b){ + callback(b,r,e); + }); + }; + return this; } diff --git a/routes/api.js b/routes/api.js index 1249bba..53ccd35 100644 --- a/routes/api.js +++ b/routes/api.js @@ -44,11 +44,11 @@ var workerCreate = function(){ image: '17575764' }, function(data){ data = JSON.parse(data); + setTimeout(function(dopletNewID){ + checkDroplet(dopletNewID) + }, 60000, data.droplet.id); doapi.dropletSetTag('clworker', data.droplet.id, function(d){ console.log(d) - setTimeout(function(dopletNewID){ - checkDroplet(dopletNewID) - }, 60000, data.droplet.id); }); }); @@ -176,7 +176,7 @@ var initWorkers = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); data['droplets'].forEach(function(worker){ - + console.log('found old droplet, killing it'); doapi.dropletDestroy(worker.id, function(){}); }); checkWorkersBalance(); From 37fae2e41353a7bba1396e33ee134eb32a173b4f Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 13:43:04 -0400 Subject: [PATCH 120/175] stuff --- routes/api.js | 255 +++++++++++++++++++++++++------------------------- 1 file changed, 129 insertions(+), 126 deletions(-) diff --git a/routes/api.js b/routes/api.js index 53ccd35..3b1d190 100644 --- a/routes/api.js +++ b/routes/api.js @@ -17,84 +17,143 @@ var isCheckingWorkers = false; var dopletNewID = 0; -var checkDroplet = function(id, time){ - time = time || 5000; - doapi.dropletInfo(id, function(data){ - var worker = JSON.parse(data)['droplet']; - if(worker.status == 'active'){ - console.log('Droplet is now active, starting runners in 20 seconds') - setTimeout(function(worker){ - console.log('Ready to start runners!') - startRunners(workers[workers.push(makeWorkerObj(worker))-1]) - isCheckingWorkers = false; - }, 20000, worker); - return true; - }else{ - console.log('Worker not ready, check again in ', time, 'MS'); - setTimeout(function(){ - checkDroplet(id) - }, time); - } - }); -}; +var workers = (function(){ + var workers = []; -var workerCreate = function(){ - doapi.dropletCreate({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), - image: '17575764' - }, function(data){ - data = JSON.parse(data); - setTimeout(function(dopletNewID){ - checkDroplet(dopletNewID) - }, 60000, data.droplet.id); - doapi.dropletSetTag('clworker', data.droplet.id, function(d){ - console.log(d) + workers.checkDroplet = function(id, time){ + time = time || 5000; + doapi.dropletInfo(id, function(data){ + var worker = JSON.parse(data)['droplet']; + if(worker.status == 'active'){ + console.log('Droplet is now active, starting runners in 20 seconds') + setTimeout(function(worker){ + console.log('Ready to start runners!') + workers.startRunners(workers[workers.push(makeWorkerObj(worker))-1]) + isCheckingWorkers = false; + }, 20000, worker); + return true; + }else{ + console.log('Worker not ready, check again in ', time, 'MS'); + setTimeout(function(){ + workers.checkDroplet(id); + }, time); + } }); - }); + }; -}; + workers.create = function(){ + doapi.dropletCreate({ + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), + image: '17575764' + }, function(data){ + data = JSON.parse(data); + setTimeout(function(dopletNewID){ + workers.checkDroplet(dopletNewID); + }, 60000, data.droplet.id); + doapi.dropletSetTag('clworker', data.droplet.id, function(){}); + }); -var workerDestroy = function(worker){ - worker = worker || workers.pop(); - doapi.dropletDestroy(worker.id, function(){}); - checkWorkersBalance(); -}; + }; -var checkWorkersBalance = function(){ - if(isCheckingWorkers) return false; - isCheckingWorkers = true; - var changed = false; - console.log('checking balance'); + workers.destroy = function(worker){ + worker = worker || workers.pop(); + doapi.dropletDestroy(worker.id, function(){}); + workers.checkBalance(); + }; - if(workers.length < 2){ - console.log('less then 2 workers, starting a droplet'); - for(var i=2; i--;) workerCreate(); - return ; - } - if(workers[workers.length-1].usedrunner !== 0){ - console.log('last droplet has no free runners, starting droplet'); - return workerCreate(); - } - if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ - console.log('Last 2 runners not used, killing last runner'); - workerDestroy(); - } - - for(let worker of workers){ - if(worker.availrunners.length === 0 && worker.usedrunner === 0){ - workerDestroy(worker); - changed = true; + workers.makeWorkerObj = function(worker){ + worker.networks.v4.forEach(function(value){ + worker[value.type+'IP'] = value.ip_address; + }); + worker.availrunners = []; + worker.ip = worker.privateIP; + worker.usedrunner = 0; + worker.index = workers.length, + worker.getRunner = function(){ + if(this.availrunners.length === 0) return false; + console.log('geting runner from ', worker.name, ' aval length ', this.availrunners.length); + var runner = this.availrunners.pop(); + this.usedrunner++; + label2runner[runner.label] = runner; + + return runner; } - } + return worker; + }; - console.log('stopping workers balancing check'); - isCheckingWorkers = false; - if(changed) setTimeout(function(){ - checkWorkersBalance(); - }, 3000); -}; + workers.destroyOld = function(){ + doapi.dropletsByTag('clworker', function(data){ + data = JSON.parse(data); + data['droplets'].forEach(function(worker){ + console.log('found old droplet, killing it'); + doapi.dropletDestroy(worker.id, function(){}); + }); + workers.checkBalance(); + }); + }; -// var start + workers.startRunners = function(worker, stopPercent){ + console.log('starting runners on', worker.name) + stopPercent = stopPercent || 80; + ramPercentUsed(worker.ip, function(usedMemPercent){ + if(usedMemPercent < stopPercent ){ + var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); + return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ + if( !data.ip ) return setTimeout(startRunners(worker),0); + console.log('started runner') + + worker.availrunners.push({ + ip: data.ip, + name: name, + worker: worker, + label: worker.name + ':' + name + }); + return setTimeout(startRunners(worker, stopPercent), 0); + }); + }else{ + setTimeout(workers.checkBalance, 10000); + console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); + } + }); + }; + + workers.checkBalance = function(){ + if(isCheckingWorkers) return false; + isCheckingWorkers = true; + var changed = false; + console.log('checking balance'); + + if(workers.length < 2){ + console.log('less then 2 workers, starting a droplet'); + for(var i=2; i--;) workers.workerCreate(); + return ; + } + if(workers[workers.length-1].usedrunner !== 0){ + console.log('last droplet has no free runners, starting droplet'); + return workers.workerCreate(); + } + if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + console.log('Last 2 runners not used, killing last runner'); + workers.workerDestroy(); + } + + for(let worker of workers){ + if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + workers.workerDestroy(worker); + changed = true; + } + } + + console.log('stopping workers balancing check'); + isCheckingWorkers = false; + if(changed) setTimeout(function(){ + workers.checkBalance(); + }, 3000); + }; + + return workers; + +})(); var ramPercentUsed = function(ip, callback){ @@ -113,7 +172,7 @@ var runnerFree = function(runner){ } delete label2runner[runner.label]; - startRunners(runner.worker); + workers.startRunners(runner.worker); }; var lxcTimeout = function(runner, time){ @@ -152,37 +211,6 @@ var run = function(req, res, runner){ }); }; -var makeWorkerObj = function(worker){ - worker.networks.v4.forEach(function(value){ - worker[value.type+'IP'] = value.ip_address; - }); - worker.availrunners = []; - worker.ip = worker.privateIP; - worker.usedrunner = 0; - worker.index = workers.length, - worker.getRunner = function(){ - if(this.availrunners.length === 0) return false; - console.log('geting runner from ', worker.name, ' aval length ', this.availrunners.length); - var runner = this.availrunners.pop(); - this.usedrunner++; - label2runner[runner.label] = runner; - - return runner; - } - return worker; -}; - -var initWorkers = function(){ - doapi.dropletsByTag('clworker', function(data){ - data = JSON.parse(data); - data['droplets'].forEach(function(worker){ - console.log('found old droplet, killing it'); - doapi.dropletDestroy(worker.id, function(){}); - }); - checkWorkersBalance(); - }); -}; - var getAvailrunner = function(runner){ for(let worker of workers){ console.log('checking ', worker.name, ' with ', worker.availrunners.length, ' free workers'); @@ -195,31 +223,6 @@ var getAvailrunner = function(runner){ return false; }; -var startRunners = function(worker, stopPercent){ - console.log('starting runners on', worker.name) - stopPercent = stopPercent || 80; - ramPercentUsed(worker.ip, function(usedMemPercent){ - if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); - return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if( !data.ip ) return setTimeout(startRunners(worker),0); - console.log('started runner') - - worker.availrunners.push({ - ip: data.ip, - name: name, - worker: worker, - label: worker.name + ':' + name - }); - return setTimeout(startRunners(worker, stopPercent), 0); - }); - }else{ - setTimeout(checkWorkersBalance, 10000); - console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); - } - }); -}; - initWorkers(); // router.get('/start/:name', function(req, res, next){ From f0d6016a63612398128db11423688179726f5394 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 13:44:34 -0400 Subject: [PATCH 121/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 3b1d190..83d980b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -223,7 +223,7 @@ var getAvailrunner = function(runner){ return false; }; -initWorkers(); +workers.destroyOld(); // router.get('/start/:name', function(req, res, next){ // return lxc.start(req.params.name, function(data){ From 848f4391228245264753b1bea4d7f48428255278 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 13:47:36 -0400 Subject: [PATCH 122/175] stuff --- routes/api.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routes/api.js b/routes/api.js index 83d980b..cb42f58 100644 --- a/routes/api.js +++ b/routes/api.js @@ -125,21 +125,21 @@ var workers = (function(){ if(workers.length < 2){ console.log('less then 2 workers, starting a droplet'); - for(var i=2; i--;) workers.workerCreate(); + for(var i=2; i--;) workers.create(); return ; } if(workers[workers.length-1].usedrunner !== 0){ console.log('last droplet has no free runners, starting droplet'); - return workers.workerCreate(); + return workers.create(); } if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner'); - workers.workerDestroy(); + workers.destroy(); } for(let worker of workers){ if(worker.availrunners.length === 0 && worker.usedrunner === 0){ - workers.workerDestroy(worker); + workers.destroy(worker); changed = true; } } From 05b6a477f45fb2fcf4213549dc381708b88ca082 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 13:50:19 -0400 Subject: [PATCH 123/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index cb42f58..a569f67 100644 --- a/routes/api.js +++ b/routes/api.js @@ -28,7 +28,7 @@ var workers = (function(){ console.log('Droplet is now active, starting runners in 20 seconds') setTimeout(function(worker){ console.log('Ready to start runners!') - workers.startRunners(workers[workers.push(makeWorkerObj(worker))-1]) + workers.startRunners(workers[workers.push(workers.makeWorkerObj(worker))-1]) isCheckingWorkers = false; }, 20000, worker); return true; From f63b6c3acd0686e7c07d2938ada4edef56bd41b1 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 14:03:58 -0400 Subject: [PATCH 124/175] stuff --- routes/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index a569f67..3b3040a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -28,7 +28,7 @@ var workers = (function(){ console.log('Droplet is now active, starting runners in 20 seconds') setTimeout(function(worker){ console.log('Ready to start runners!') - workers.startRunners(workers[workers.push(workers.makeWorkerObj(worker))-1]) + workers.workers.startRunners(workers[workers.push(workers.makeWorkerObj(worker))-1]) isCheckingWorkers = false; }, 20000, worker); return true; @@ -99,7 +99,7 @@ var workers = (function(){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if( !data.ip ) return setTimeout(startRunners(worker),0); + if( !data.ip ) return setTimeout(workers.startRunners(worker),0); console.log('started runner') worker.availrunners.push({ @@ -108,7 +108,7 @@ var workers = (function(){ worker: worker, label: worker.name + ':' + name }); - return setTimeout(startRunners(worker, stopPercent), 0); + return setTimeout(workers.startRunners(worker, stopPercent), 0); }); }else{ setTimeout(workers.checkBalance, 10000); From b1e3e23d9153a2ea6f6637f9df2c5cb98cb010c7 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 14:06:24 -0400 Subject: [PATCH 125/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 3b3040a..613abef 100644 --- a/routes/api.js +++ b/routes/api.js @@ -28,7 +28,7 @@ var workers = (function(){ console.log('Droplet is now active, starting runners in 20 seconds') setTimeout(function(worker){ console.log('Ready to start runners!') - workers.workers.startRunners(workers[workers.push(workers.makeWorkerObj(worker))-1]) + workers.startRunners(workers[workers.push(workers.makeWorkerObj(worker))-1]) isCheckingWorkers = false; }, 20000, worker); return true; From 23f7cf1f68cf694e69b1e534af781e7aaff7f8ba Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 18:41:11 -0400 Subject: [PATCH 126/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 613abef..67655bf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -202,6 +202,7 @@ var run = function(req, res, runner){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) if(error) return false; + console.log('body', body); body = JSON.parse(body); body['ip'] = getAvailrunner(runner).label; From ce96c7cb87a49c53aaf2db8bf53ffe4abe866dd7 Mon Sep 17 00:00:00 2001 From: william Date: Sun, 29 May 2016 18:53:30 -0400 Subject: [PATCH 127/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 67655bf..63829cb 100644 --- a/routes/api.js +++ b/routes/api.js @@ -294,7 +294,7 @@ router.get('/stop/:name', function(req, res, next){ router.get('/liststuff', function(req, res, next){ var obj = util.inspect(workers, {depth: 4}); - res.send(obj); + res.send("
"+obj+"
"); }); router.post('/run/:ip?', function doRun(req, res, next){ From 9b7936ee3a2f15d4e5fe891312e118249c6c19f9 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 15:09:17 -0400 Subject: [PATCH 128/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 63829cb..9b7c9ad 100644 --- a/routes/api.js +++ b/routes/api.js @@ -201,7 +201,7 @@ var run = function(req, res, runner){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) - if(error) return false; + if(res.statusCode !== 200 || error) return run(req, res, getAvailrunner()); console.log('body', body); body = JSON.parse(body); From f714a6e1796d9d4c7aefc8e6cbef5ac0b57f755e Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 15:12:24 -0400 Subject: [PATCH 129/175] stuff --- routes/api.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 9b7c9ad..293c094 100644 --- a/routes/api.js +++ b/routes/api.js @@ -201,8 +201,7 @@ var run = function(req, res, runner){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) - if(res.statusCode !== 200 || error) return run(req, res, getAvailrunner()); - console.log('body', body); + if(response.statusCode !== 200 || error) return run(req, res, getAvailrunner()); body = JSON.parse(body); body['ip'] = getAvailrunner(runner).label; From 232e9b72bb0e4462c4cc3db1ae4ff24d285dcc58 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 15:41:30 -0400 Subject: [PATCH 130/175] stuff --- routes/api.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routes/api.js b/routes/api.js index 293c094..ce52edb 100644 --- a/routes/api.js +++ b/routes/api.js @@ -139,6 +139,7 @@ var workers = (function(){ for(let worker of workers){ if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + console.log('found zombie worker, destroying') workers.destroy(worker); changed = true; } @@ -217,6 +218,7 @@ var getAvailrunner = function(runner){ if(worker.availrunners.length === 0) continue; if(runner && runner.worker.index <= worker.index) break; if(runner) runnerFree(runner); + workers.checkBalance(); return worker.getRunner(); } if(runner) return runner; From 651f3835e214043ecd84b4b44883d8bde53215f3 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 15:43:49 -0400 Subject: [PATCH 131/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index ce52edb..30b795f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -84,6 +84,7 @@ var workers = (function(){ workers.destroyOld = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); + console.log('dropletsByTag data:',data); data['droplets'].forEach(function(worker){ console.log('found old droplet, killing it'); doapi.dropletDestroy(worker.id, function(){}); From 18ff2ee3be6a2a8f6264b394692761284fd7178f Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:06:06 -0400 Subject: [PATCH 132/175] stuff --- doapi.js | 8 ++++++++ routes/api.js | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doapi.js b/doapi.js index 0959a55..8847282 100644 --- a/doapi.js +++ b/doapi.js @@ -7,12 +7,14 @@ api = function(key){ 'Content-Type': 'application/json', 'Authorization': 'Bearer '+key } + this.calls = 0; this.account = function(callback){ var options = { url: this.BASEURL+'account', headers: this.headers }; + this.calls++; return request.get(options, function(error, response, body){ return callback(body, response, error); @@ -24,6 +26,7 @@ api = function(key){ url: this.BASEURL+'droplets?tag_name='+tag, headers: this.headers }; + this.calls++; return request.get(options, function(error, response, body){ return callback(body, response, error); @@ -44,6 +47,7 @@ api = function(key){ headers: this.headers, body: JSON.stringify(data) }; + this.calls++; return request.post(options, function(error, response, body){ return callback(body, response, error); @@ -66,6 +70,7 @@ api = function(key){ headers: this.headers, body: JSON.stringify(data) }; + this.calls++; return request.post(options, function(error, response, body){ return callback(body, response, error); @@ -77,6 +82,7 @@ api = function(key){ url: this.BASEURL+'droplets/'+dropletID, headers: this.headers }; + this.calls++; return request.del(options, function(error, response, body){ callback(body, response, error); @@ -88,6 +94,7 @@ api = function(key){ url: this.BASEURL+'droplets/'+dropletID, headers: this.headers }; + this.calls++; return request.get(options, function(error, response, body){ callback(body, response, error); @@ -99,6 +106,7 @@ api = function(key){ url: this.BASEURL+'tags', headers: this.headers }; + this.calls++; return request.get(options, function(e,r,b){ callback(b,r,e); diff --git a/routes/api.js b/routes/api.js index 30b795f..56c5638 100644 --- a/routes/api.js +++ b/routes/api.js @@ -296,7 +296,7 @@ router.get('/stop/:name', function(req, res, next){ router.get('/liststuff', function(req, res, next){ var obj = util.inspect(workers, {depth: 4}); - res.send("
"+obj+"
"); + res.send("

Workers

"+obj+"

label2runner

"util.inspect(label2runner)+'

DO calls

'+doapi.calls); }); router.post('/run/:ip?', function doRun(req, res, next){ From d1a87a033b7238316f13095b7c07141581df491d Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:06:36 -0400 Subject: [PATCH 133/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 56c5638..2ca0fcf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -296,7 +296,7 @@ router.get('/stop/:name', function(req, res, next){ router.get('/liststuff', function(req, res, next){ var obj = util.inspect(workers, {depth: 4}); - res.send("

Workers

"+obj+"

label2runner

"util.inspect(label2runner)+'

DO calls

'+doapi.calls); + res.send("

Workers

"+obj+"

label2runner

"+util.inspect(label2runner)+'

DO calls

'+doapi.calls); }); router.post('/run/:ip?', function doRun(req, res, next){ From 4c2a4b7657760b79337b4411e59e7cafd8ab6699 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:21:22 -0400 Subject: [PATCH 134/175] stuff --- routes/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 2ca0fcf..c82e3c9 100644 --- a/routes/api.js +++ b/routes/api.js @@ -124,8 +124,8 @@ var workers = (function(){ var changed = false; console.log('checking balance'); - if(workers.length < 2){ - console.log('less then 2 workers, starting a droplet'); + if(workers.length < 3){ + console.log('less then 3 workers, starting a droplet'); for(var i=2; i--;) workers.create(); return ; } @@ -133,7 +133,7 @@ var workers = (function(){ console.log('last droplet has no free runners, starting droplet'); return workers.create(); } - if(workers.length > 2 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + if(workers.length > 3 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner'); workers.destroy(); } @@ -226,6 +226,7 @@ var getAvailrunner = function(runner){ return false; }; +setInterval(workers.checkBalance, 30000); workers.destroyOld(); // router.get('/start/:name', function(req, res, next){ From d22a79c81e1976b98566de0c6b3c11f01e04dba9 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:28:07 -0400 Subject: [PATCH 135/175] stuff --- routes/api.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/routes/api.js b/routes/api.js index c82e3c9..aa0673c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -89,7 +89,7 @@ var workers = (function(){ console.log('found old droplet, killing it'); doapi.dropletDestroy(worker.id, function(){}); }); - workers.checkBalance(); + // workers.checkBalance(); }); }; @@ -122,20 +122,22 @@ var workers = (function(){ if(isCheckingWorkers) return false; isCheckingWorkers = true; var changed = false; + var minWorkers = 3; console.log('checking balance'); - if(workers.length < 3){ + if(workers.length < minWorkers){ console.log('less then 3 workers, starting a droplet'); - for(var i=2; i--;) workers.create(); + for(var i=minWorkers; i--;) workers.create(); return ; } if(workers[workers.length-1].usedrunner !== 0){ console.log('last droplet has no free runners, starting droplet'); return workers.create(); } - if(workers.length > 3 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + if(workers.length > minWorkers && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner'); workers.destroy(); + changed = true; } for(let worker of workers){ @@ -150,7 +152,7 @@ var workers = (function(){ isCheckingWorkers = false; if(changed) setTimeout(function(){ workers.checkBalance(); - }, 3000); + }, 5000); }; return workers; From 6c2a8d48bfa4459cbca151cb8db00ae4721c13cf Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:31:46 -0400 Subject: [PATCH 136/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index aa0673c..8d90ade 100644 --- a/routes/api.js +++ b/routes/api.js @@ -141,6 +141,7 @@ var workers = (function(){ } for(let worker of workers){ + if(worker.length <= 3) break; if(worker.availrunners.length === 0 && worker.usedrunner === 0){ console.log('found zombie worker, destroying') workers.destroy(worker); From 136bf048252d658378556f6f2265119515f7ec8a Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:34:54 -0400 Subject: [PATCH 137/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 8d90ade..c23cb53 100644 --- a/routes/api.js +++ b/routes/api.js @@ -58,7 +58,7 @@ var workers = (function(){ workers.destroy = function(worker){ worker = worker || workers.pop(); doapi.dropletDestroy(worker.id, function(){}); - workers.checkBalance(); + // workers.checkBalance(); }; workers.makeWorkerObj = function(worker){ @@ -89,7 +89,7 @@ var workers = (function(){ console.log('found old droplet, killing it'); doapi.dropletDestroy(worker.id, function(){}); }); - // workers.checkBalance(); + workers.checkBalance(); }); }; From f9482b348368e5e31b15e43003dd57ae8966734f Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:54:06 -0400 Subject: [PATCH 138/175] stuff --- routes/api.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index c23cb53..259a2fe 100644 --- a/routes/api.js +++ b/routes/api.js @@ -28,7 +28,7 @@ var workers = (function(){ console.log('Droplet is now active, starting runners in 20 seconds') setTimeout(function(worker){ console.log('Ready to start runners!') - workers.startRunners(workers[workers.push(workers.makeWorkerObj(worker))-1]) + workers.startRunners(worker, true) isCheckingWorkers = false; }, 20000, worker); return true; @@ -93,7 +93,7 @@ var workers = (function(){ }); }; - workers.startRunners = function(worker, stopPercent){ + workers.startRunners = function(worker, new,stopPercent){ console.log('starting runners on', worker.name) stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ @@ -102,6 +102,7 @@ var workers = (function(){ return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if( !data.ip ) return setTimeout(workers.startRunners(worker),0); console.log('started runner') + if(new) worker = workers[workers.push(workers.makeWorkerObj(worker))-1] worker.availrunners.push({ ip: data.ip, From b8f72ef13c7350fe7c46e542205adc95f54f6c93 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 16:55:48 -0400 Subject: [PATCH 139/175] stuff --- routes/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 259a2fe..1373c54 100644 --- a/routes/api.js +++ b/routes/api.js @@ -93,16 +93,16 @@ var workers = (function(){ }); }; - workers.startRunners = function(worker, new,stopPercent){ + workers.startRunners = function(worker, newWorker, stopPercent){ console.log('starting runners on', worker.name) stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ - if( !data.ip ) return setTimeout(workers.startRunners(worker),0); + if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner') - if(new) worker = workers[workers.push(workers.makeWorkerObj(worker))-1] + if(newWorker) worker = workers[workers.push(workers.makeWorkerObj(worker))-1] worker.availrunners.push({ ip: data.ip, From c971b609eeef6a19caeeb55ae98e7b0e2791bacc Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:03:21 -0400 Subject: [PATCH 140/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 1373c54..c494ad5 100644 --- a/routes/api.js +++ b/routes/api.js @@ -97,6 +97,7 @@ var workers = (function(){ console.log('starting runners on', worker.name) stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ + console.log('using ', usedMemPercent, ' on ', worker.name, ' continueing'); if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ From a77d00aab444ef8bf6a5b762b7b57fb916496b16 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:06:03 -0400 Subject: [PATCH 141/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index c494ad5..84cc987 100644 --- a/routes/api.js +++ b/routes/api.js @@ -94,7 +94,7 @@ var workers = (function(){ }; workers.startRunners = function(worker, newWorker, stopPercent){ - console.log('starting runners on', worker.name) + console.log('starting runners on', worker.name, worker.ip) stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ console.log('using ', usedMemPercent, ' on ', worker.name, ' continueing'); From 421a8ec78b2873e2415a0f4e6a0809798b874d16 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:10:32 -0400 Subject: [PATCH 142/175] stuff --- routes/api.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 84cc987..ca12ffd 100644 --- a/routes/api.js +++ b/routes/api.js @@ -28,7 +28,7 @@ var workers = (function(){ console.log('Droplet is now active, starting runners in 20 seconds') setTimeout(function(worker){ console.log('Ready to start runners!') - workers.startRunners(worker, true) + workers.startRunners(workers.makeWorkerObj(worker), true) isCheckingWorkers = false; }, 20000, worker); return true; @@ -84,7 +84,6 @@ var workers = (function(){ workers.destroyOld = function(){ doapi.dropletsByTag('clworker', function(data){ data = JSON.parse(data); - console.log('dropletsByTag data:',data); data['droplets'].forEach(function(worker){ console.log('found old droplet, killing it'); doapi.dropletDestroy(worker.id, function(){}); @@ -103,7 +102,7 @@ var workers = (function(){ return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner') - if(newWorker) worker = workers[workers.push(workers.makeWorkerObj(worker))-1] + if(newWorker) worker = workers[workers.push(worker)-1] worker.availrunners.push({ ip: data.ip, From eec89027a47ae76c38aabbdc6eddcce445a6e0c9 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:18:36 -0400 Subject: [PATCH 143/175] stuff --- routes/api.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index ca12ffd..f175837 100644 --- a/routes/api.js +++ b/routes/api.js @@ -136,9 +136,8 @@ var workers = (function(){ return workers.create(); } if(workers.length > minWorkers && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ - console.log('Last 2 runners not used, killing last runner'); - workers.destroy(); - changed = true; + console.log('Last 2 runners not used, killing last runner', workers.length); + return workers.destroy(); } for(let worker of workers){ From b8dfaff8613802d12d69973e56bd369925460fa7 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:27:21 -0400 Subject: [PATCH 144/175] stuff --- routes/api.js | 2 +- testAPI.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 testAPI.py diff --git a/routes/api.js b/routes/api.js index f175837..ca9afff 100644 --- a/routes/api.js +++ b/routes/api.js @@ -206,7 +206,7 @@ var run = function(req, res, runner){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) - if(response.statusCode !== 200 || error) return run(req, res, getAvailrunner()); + if(error || response.statusCode !== 200) return run(req, res, getAvailrunner()); body = JSON.parse(body); body['ip'] = getAvailrunner(runner).label; diff --git a/testAPI.py b/testAPI.py new file mode 100644 index 0000000..9ae66a7 --- /dev/null +++ b/testAPI.py @@ -0,0 +1,10 @@ +import requests as r +import time + +def testAPI(times): + errors = 0 + + for i in range(times): + res = r.post('http://codeland.bytedev.co:2000/api/run', data={'code': 'pwd'}) + if res.status_code != 200: errors +=1 + print('errors ', errors) \ No newline at end of file From be7f5187783d42fe2aed369772728de91c918d7e Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:50:21 -0400 Subject: [PATCH 145/175] stuff --- routes/api.js | 4 ++-- testAPI.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/routes/api.js b/routes/api.js index ca9afff..1738d4c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -181,7 +181,7 @@ var runnerFree = function(runner){ }; var lxcTimeout = function(runner, time){ - time = time || 60000 // 900000; // 15 minutes + time = time || 900000; // 15 minutes if(runner.hasOwnProperty('timeout')){ clearTimeout(runner.timeout); @@ -305,7 +305,7 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route') - var runner = label2runner[req.params.ip] || getAvailrunner(); + var runner = label2runner[req.params.ip] || false; console.log('') return run(req, res, runner); }); diff --git a/testAPI.py b/testAPI.py index 9ae66a7..decd1ef 100644 --- a/testAPI.py +++ b/testAPI.py @@ -1,10 +1,19 @@ import requests as r import time -def testAPI(times): +def testAPI(times=100, sleep=2): errors = 0 for i in range(times): - res = r.post('http://codeland.bytedev.co:2000/api/run', data={'code': 'pwd'}) - if res.status_code != 200: errors +=1 + try: + res = r.post( + 'http://codeland.bytedev.co:2000/api/run', + data={'code': 'pwd'} + ) + if res.status_code != 200: errors += 1 + print(i, res.status_code) + except: + print('caught error') + errors += 1 + time.sleep(sleep) print('errors ', errors) \ No newline at end of file From 71ab12c64e595ab792823b42bdc9fe1978779946 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 17:56:52 -0400 Subject: [PATCH 146/175] stuff --- routes/api.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routes/api.js b/routes/api.js index 1738d4c..5de348f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -203,13 +203,14 @@ var run = function(req, res, runner){ code: req.body.code }) }; + runner = getAvailrunner(runner); return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) if(error || response.statusCode !== 200) return run(req, res, getAvailrunner()); body = JSON.parse(body); - body['ip'] = getAvailrunner(runner).label; + body['ip'] = runner.label; lxcTimeout(runner); return res.json(body); @@ -305,9 +306,8 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route') - var runner = label2runner[req.params.ip] || false; - console.log('') - return run(req, res, runner); + + return run(req, res, label2runner[req.params.ip]); }); module.exports = router; From 5a5c453ea31157dd99418683830d512c47f27e29 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:03:11 -0400 Subject: [PATCH 147/175] stuff --- routes/api.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/routes/api.js b/routes/api.js index 5de348f..505cf9c 100644 --- a/routes/api.js +++ b/routes/api.js @@ -21,7 +21,7 @@ var workers = (function(){ var workers = []; workers.checkDroplet = function(id, time){ - time = time || 5000; + time = time || 10000; doapi.dropletInfo(id, function(data){ var worker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ @@ -49,7 +49,7 @@ var workers = (function(){ data = JSON.parse(data); setTimeout(function(dopletNewID){ workers.checkDroplet(dopletNewID); - }, 60000, data.droplet.id); + }, 70000, data.droplet.id); doapi.dropletSetTag('clworker', data.droplet.id, function(){}); }); @@ -88,7 +88,6 @@ var workers = (function(){ console.log('found old droplet, killing it'); doapi.dropletDestroy(worker.id, function(){}); }); - workers.checkBalance(); }); }; @@ -96,7 +95,6 @@ var workers = (function(){ console.log('starting runners on', worker.name, worker.ip) stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ - console.log('using ', usedMemPercent, ' on ', worker.name, ' continueing'); if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ @@ -113,7 +111,6 @@ var workers = (function(){ return setTimeout(workers.startRunners(worker, stopPercent), 0); }); }else{ - setTimeout(workers.checkBalance, 10000); console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); } }); @@ -145,15 +142,11 @@ var workers = (function(){ if(worker.availrunners.length === 0 && worker.usedrunner === 0){ console.log('found zombie worker, destroying') workers.destroy(worker); - changed = true; } } console.log('stopping workers balancing check'); isCheckingWorkers = false; - if(changed) setTimeout(function(){ - workers.checkBalance(); - }, 5000); }; return workers; @@ -204,7 +197,7 @@ var run = function(req, res, runner){ }) }; runner = getAvailrunner(runner); - + return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) if(error || response.statusCode !== 200) return run(req, res, getAvailrunner()); @@ -307,7 +300,7 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route') - return run(req, res, label2runner[req.params.ip]); + return run(req, res, label2runner[req.params.ip] || null); }); module.exports = router; From e8c26abd52d7a6fc55cb3b2d1b5d7f41be222aa7 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:03:59 -0400 Subject: [PATCH 148/175] stuff --- routes/api.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 505cf9c..fb15561 100644 --- a/routes/api.js +++ b/routes/api.js @@ -58,7 +58,6 @@ var workers = (function(){ workers.destroy = function(worker){ worker = worker || workers.pop(); doapi.dropletDestroy(worker.id, function(){}); - // workers.checkBalance(); }; workers.makeWorkerObj = function(worker){ @@ -216,7 +215,6 @@ var getAvailrunner = function(runner){ if(worker.availrunners.length === 0) continue; if(runner && runner.worker.index <= worker.index) break; if(runner) runnerFree(runner); - workers.checkBalance(); return worker.getRunner(); } if(runner) return runner; From 1624d9f7ffa64a207d5ec99948865b5f345af7f4 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:11:00 -0400 Subject: [PATCH 149/175] stuff --- routes/api.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index fb15561..d1830b9 100644 --- a/routes/api.js +++ b/routes/api.js @@ -195,7 +195,6 @@ var run = function(req, res, runner){ code: req.body.code }) }; - runner = getAvailrunner(runner); return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) @@ -297,8 +296,8 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route') - - return run(req, res, label2runner[req.params.ip] || null); + var runner = getAvailrunner(label2runner[req.params.ip]); + return run(req, res, runner); }); module.exports = router; From cfc6d406a1ab5c2dbdb0ebf0ac1f68bff2f75717 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:11:52 -0400 Subject: [PATCH 150/175] stuff --- routes/api.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index d1830b9..7c1ccbb 100644 --- a/routes/api.js +++ b/routes/api.js @@ -57,7 +57,9 @@ var workers = (function(){ workers.destroy = function(worker){ worker = worker || workers.pop(); - doapi.dropletDestroy(worker.id, function(){}); + doapi.dropletDestroy(worker.id, function(){ + isCheckingWorkers = false; + }); }; workers.makeWorkerObj = function(worker){ From 4b4ab75aaa61ca5b2573a12d21d992b169db7ce2 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:20:47 -0400 Subject: [PATCH 151/175] stuff --- routes/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7c1ccbb..a34c915 100644 --- a/routes/api.js +++ b/routes/api.js @@ -126,7 +126,7 @@ var workers = (function(){ if(workers.length < minWorkers){ console.log('less then 3 workers, starting a droplet'); - for(var i=minWorkers; i--;) workers.create(); + for(var i=minWorkers-workers.length; i--;) workers.create(); return ; } if(workers[workers.length-1].usedrunner !== 0){ @@ -221,8 +221,9 @@ var getAvailrunner = function(runner){ if(runner) return runner; return false; }; - -setInterval(workers.checkBalance, 30000); +setTimeout(function(){ + setInterval(workers.checkBalance, 30000); +}, 180000); workers.destroyOld(); // router.get('/start/:name', function(req, res, next){ From 3e0d34a2f5a9d0e6c9d8402af18180b09411ae2d Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:35:00 -0400 Subject: [PATCH 152/175] stuff --- routes/api.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/routes/api.js b/routes/api.js index a34c915..adf955b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -56,7 +56,7 @@ var workers = (function(){ }; workers.destroy = function(worker){ - worker = worker || workers.pop(); + var worker = worker || workers.pop(); doapi.dropletDestroy(worker.id, function(){ isCheckingWorkers = false; }); @@ -93,14 +93,14 @@ var workers = (function(){ }; workers.startRunners = function(worker, newWorker, stopPercent){ - console.log('starting runners on', worker.name, worker.ip) + // console.log('starting runners on', worker.name, worker.ip) stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); - console.log('started runner') + console.log('started runner on', worker.name) if(newWorker) worker = workers[workers.push(worker)-1] worker.availrunners.push({ @@ -112,7 +112,7 @@ var workers = (function(){ return setTimeout(workers.startRunners(worker, stopPercent), 0); }); }else{ - console.log('using', usedMemPercent, 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); + console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); } }); }; @@ -222,6 +222,7 @@ var getAvailrunner = function(runner){ return false; }; setTimeout(function(){ + console.log('Starting balance checking in 30 seconds') setInterval(workers.checkBalance, 30000); }, 180000); workers.destroyOld(); From 2237d3cac31f671edfa756decbb3a47fc0267e9f Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:37:08 -0400 Subject: [PATCH 153/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index adf955b..16a61b6 100644 --- a/routes/api.js +++ b/routes/api.js @@ -226,6 +226,7 @@ setTimeout(function(){ setInterval(workers.checkBalance, 30000); }, 180000); workers.destroyOld(); +workers.checkBalance(); // router.get('/start/:name', function(req, res, next){ // return lxc.start(req.params.name, function(data){ From fd4405bca502fe8b422092ac4cd9f9edcfefbd23 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:46:29 -0400 Subject: [PATCH 154/175] stuff --- routes/api.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 16a61b6..a7495fb 100644 --- a/routes/api.js +++ b/routes/api.js @@ -20,6 +20,11 @@ var dopletNewID = 0; var workers = (function(){ var workers = []; + workers.push = function(item){ + console.log('pushing item ', item, ' to workers') + return Array.prototype.push.call(this, item); + }; + workers.checkDroplet = function(id, time){ time = time || 10000; doapi.dropletInfo(id, function(data){ @@ -109,7 +114,7 @@ var workers = (function(){ worker: worker, label: worker.name + ':' + name }); - return setTimeout(workers.startRunners(worker, stopPercent), 0); + return setTimeout(workers.startRunners(worker, false ,stopPercent), 0); }); }else{ console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name); From 7c97bed5b03bb6a2908acd4a780e1527097cb0fb Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 18:54:36 -0400 Subject: [PATCH 155/175] stuff --- routes/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/api.js b/routes/api.js index a7495fb..41a19f4 100644 --- a/routes/api.js +++ b/routes/api.js @@ -134,11 +134,11 @@ var workers = (function(){ for(var i=minWorkers-workers.length; i--;) workers.create(); return ; } - if(workers[workers.length-1].usedrunner !== 0){ + if(workers[workers.length-2].usedrunner !== 0){ console.log('last droplet has no free runners, starting droplet'); return workers.create(); } - if(workers.length > minWorkers && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + if(workers.length > minWorkers && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner', workers.length); return workers.destroy(); } @@ -228,7 +228,7 @@ var getAvailrunner = function(runner){ }; setTimeout(function(){ console.log('Starting balance checking in 30 seconds') - setInterval(workers.checkBalance, 30000); + setInterval(workers.checkBalance, 15000); }, 180000); workers.destroyOld(); workers.checkBalance(); From 9072167e65bfaf5ca3fa22799b797310f04c4ee4 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 19:45:51 -0400 Subject: [PATCH 156/175] stuff --- routes/api.js | 47 ++++++++++++++++++++--------------------------- testAPI.py | 2 +- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/routes/api.js b/routes/api.js index 41a19f4..ed254c3 100644 --- a/routes/api.js +++ b/routes/api.js @@ -15,31 +15,25 @@ var label2runner = {}; var workers = []; var isCheckingWorkers = false; -var dopletNewID = 0; - var workers = (function(){ var workers = []; - workers.push = function(item){ - console.log('pushing item ', item, ' to workers') - return Array.prototype.push.call(this, item); - }; - workers.checkDroplet = function(id, time){ time = time || 10000; - doapi.dropletInfo(id, function(data){ + + return doapi.dropletInfo(id, function(data){ var worker = JSON.parse(data)['droplet']; if(worker.status == 'active'){ - console.log('Droplet is now active, starting runners in 20 seconds') - setTimeout(function(worker){ - console.log('Ready to start runners!') - workers.startRunners(workers.makeWorkerObj(worker), true) - isCheckingWorkers = false; + console.log('Droplet is now active, starting runners in 20 seconds'); + + return setTimeout(function(worker){ + console.log('Ready to start runners!'); + workers.startRunners(workers.makeWorkerObj(worker), true); }, 20000, worker); - return true; }else{ console.log('Worker not ready, check again in ', time, 'MS'); - setTimeout(function(){ + + return setTimeout(function(){ workers.checkDroplet(id); }, time); } @@ -47,24 +41,23 @@ var workers = (function(){ }; workers.create = function(){ - doapi.dropletCreate({ + return doapi.dropletCreate({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), image: '17575764' }, function(data){ data = JSON.parse(data); + setTimeout(function(dopletNewID){ - workers.checkDroplet(dopletNewID); + return workers.checkDroplet(dopletNewID); }, 70000, data.droplet.id); - doapi.dropletSetTag('clworker', data.droplet.id, function(){}); + return doapi.dropletSetTag('clworker', data.droplet.id, function(){}); }); }; workers.destroy = function(worker){ var worker = worker || workers.pop(); - doapi.dropletDestroy(worker.id, function(){ - isCheckingWorkers = false; - }); + return doapi.dropletDestroy(worker.id, function(){}); }; workers.makeWorkerObj = function(worker){ @@ -84,6 +77,7 @@ var workers = (function(){ return runner; } + return worker; }; @@ -123,9 +117,7 @@ var workers = (function(){ }; workers.checkBalance = function(){ - if(isCheckingWorkers) return false; - isCheckingWorkers = true; - var changed = false; + var minWorkers = 3; console.log('checking balance'); @@ -134,11 +126,11 @@ var workers = (function(){ for(var i=minWorkers-workers.length; i--;) workers.create(); return ; } - if(workers[workers.length-2].usedrunner !== 0){ + if(workers[workers.length-3].usedrunner !== 0){ console.log('last droplet has no free runners, starting droplet'); return workers.create(); } - if(workers.length > minWorkers && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0 && workers[workers.length-2].usedrunner === 0){ + if(workers.length > minWorkers && workers[workers.length-3].usedrunner === 0 && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0){ console.log('Last 2 runners not used, killing last runner', workers.length); return workers.destroy(); } @@ -152,7 +144,6 @@ var workers = (function(){ } console.log('stopping workers balancing check'); - isCheckingWorkers = false; }; return workers; @@ -226,10 +217,12 @@ var getAvailrunner = function(runner){ if(runner) return runner; return false; }; + setTimeout(function(){ console.log('Starting balance checking in 30 seconds') setInterval(workers.checkBalance, 15000); }, 180000); + workers.destroyOld(); workers.checkBalance(); diff --git a/testAPI.py b/testAPI.py index decd1ef..c407b8a 100644 --- a/testAPI.py +++ b/testAPI.py @@ -16,4 +16,4 @@ def testAPI(times=100, sleep=2): print('caught error') errors += 1 time.sleep(sleep) - print('errors ', errors) \ No newline at end of file + print('errors ', errors, (errors/times)*100) \ No newline at end of file From 9bb9809f512ddbf19137ffc69429587cfede2818 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 21:18:02 -0400 Subject: [PATCH 157/175] stuff --- routes/api.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/routes/api.js b/routes/api.js index ed254c3..01db099 100644 --- a/routes/api.js +++ b/routes/api.js @@ -70,14 +70,14 @@ var workers = (function(){ worker.index = workers.length, worker.getRunner = function(){ if(this.availrunners.length === 0) return false; - console.log('geting runner from ', worker.name, ' aval length ', this.availrunners.length); + console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length); var runner = this.availrunners.pop(); this.usedrunner++; label2runner[runner.label] = runner; return runner; } - + return worker; }; @@ -126,8 +126,8 @@ var workers = (function(){ for(var i=minWorkers-workers.length; i--;) workers.create(); return ; } - if(workers[workers.length-3].usedrunner !== 0){ - console.log('last droplet has no free runners, starting droplet'); + if(workers[workers.length-3].usedrunner !== 0 && workers[workers.length-2].usedrunner !== 0 && workers[workers.length-1].usedrunner !== 0){ + console.log('last 3 workers have no free runners, starting droplet'); return workers.create(); } if(workers.length > minWorkers && workers[workers.length-3].usedrunner === 0 && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0){ @@ -171,7 +171,7 @@ var runnerFree = function(runner){ }; var lxcTimeout = function(runner, time){ - time = time || 900000; // 15 minutes + time = time || 300000; // 5 minutes if(runner.hasOwnProperty('timeout')){ clearTimeout(runner.timeout); @@ -182,8 +182,9 @@ var lxcTimeout = function(runner, time){ }, time); }; -var run = function(req, res, runner){ +var run = function(req, res, runner, count){ + count = count || 0; var httpOptions = { url: 'http://' + runner.worker.ip, headers: { @@ -194,9 +195,13 @@ var run = function(req, res, runner){ }) }; + if(!runner || count > 3){ + return res.status(503).json({ error: 'No runners, try again soon.' }); + } + return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) - if(error || response.statusCode !== 200) return run(req, res, getAvailrunner()); + if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count); body = JSON.parse(body); body['ip'] = runner.label; From ded9cff1dcf547a9e5f05dbf4f6d505526563706 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 21:25:35 -0400 Subject: [PATCH 158/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index 01db099..dc29ce3 100644 --- a/routes/api.js +++ b/routes/api.js @@ -219,6 +219,7 @@ var getAvailrunner = function(runner){ if(runner) runnerFree(runner); return worker.getRunner(); } + console.log(runner); if(runner) return runner; return false; }; From a432df78795caaaa7cc279a1f6b188bfd38ae61c Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 21:46:40 -0400 Subject: [PATCH 159/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index dc29ce3..7318bb7 100644 --- a/routes/api.js +++ b/routes/api.js @@ -194,7 +194,7 @@ var run = function(req, res, runner, count){ code: req.body.code }) }; - + console.log('run', runner); if(!runner || count > 3){ return res.status(503).json({ error: 'No runners, try again soon.' }); } @@ -219,7 +219,7 @@ var getAvailrunner = function(runner){ if(runner) runnerFree(runner); return worker.getRunner(); } - console.log(runner); + console.log('getAvailrunner, none found', runner); if(runner) return runner; return false; }; From 5467209d256e6d237a6d982df5fd0476f9e5d5fd Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 22:28:59 -0400 Subject: [PATCH 160/175] stuff --- routes/api.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7318bb7..6fdb011 100644 --- a/routes/api.js +++ b/routes/api.js @@ -196,7 +196,8 @@ var run = function(req, res, runner, count){ }; console.log('run', runner); if(!runner || count > 3){ - return res.status(503).json({ error: 'No runners, try again soon.' }); + res.status(503); + return res.json({error: 'No runners, try again soon.'}); } return request.post(httpOptions, function(error, response, body){ @@ -221,6 +222,7 @@ var getAvailrunner = function(runner){ } console.log('getAvailrunner, none found', runner); if(runner) return runner; + console.log('no..') return false; }; @@ -304,7 +306,7 @@ router.get('/liststuff', function(req, res, next){ }); router.post('/run/:ip?', function doRun(req, res, next){ - console.log('hit runner route') + console.log('hit runner route'); var runner = getAvailrunner(label2runner[req.params.ip]); return run(req, res, runner); }); From 4d78659654e2a83c10b5919feae3ac6112059ca7 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 22:48:24 -0400 Subject: [PATCH 161/175] stuff --- routes/api.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/routes/api.js b/routes/api.js index 6fdb011..f747130 100644 --- a/routes/api.js +++ b/routes/api.js @@ -202,6 +202,7 @@ var run = function(req, res, runner, count){ return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) + console.log('in request'); if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count); body = JSON.parse(body); @@ -218,6 +219,7 @@ var getAvailrunner = function(runner){ if(worker.availrunners.length === 0) continue; if(runner && runner.worker.index <= worker.index) break; if(runner) runnerFree(runner); + console.log('getAvailrunner while loop', runner); return worker.getRunner(); } console.log('getAvailrunner, none found', runner); @@ -308,6 +310,7 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route'); var runner = getAvailrunner(label2runner[req.params.ip]); + console.log('route ', runner); return run(req, res, runner); }); From 67b717ace106281ec1966e53d67ea0bcc4d11d4a Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:00:09 -0400 Subject: [PATCH 162/175] stuff --- routes/api.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index f747130..7c675e1 100644 --- a/routes/api.js +++ b/routes/api.js @@ -183,8 +183,8 @@ var lxcTimeout = function(runner, time){ }; var run = function(req, res, runner, count){ - count = count || 0; + console.log('run start', count); var httpOptions = { url: 'http://' + runner.worker.ip, headers: { @@ -195,6 +195,7 @@ var run = function(req, res, runner, count){ }) }; console.log('run', runner); + if(!runner || count > 3){ res.status(503); return res.json({error: 'No runners, try again soon.'}); From aee63de4783f2a8e928b820755b71445658bcab6 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:06:45 -0400 Subject: [PATCH 163/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7c675e1..fadf3f4 100644 --- a/routes/api.js +++ b/routes/api.js @@ -184,7 +184,7 @@ var lxcTimeout = function(runner, time){ var run = function(req, res, runner, count){ count = count || 0; - console.log('run start', count); + console.log('run start', count, runner); var httpOptions = { url: 'http://' + runner.worker.ip, headers: { @@ -195,7 +195,7 @@ var run = function(req, res, runner, count){ }) }; console.log('run', runner); - + if(!runner || count > 3){ res.status(503); return res.json({error: 'No runners, try again soon.'}); From 29ad409649ccfa2eaace76e9ee99a32c7ec22607 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:14:28 -0400 Subject: [PATCH 164/175] stuff --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index fadf3f4..90ccbb4 100644 --- a/routes/api.js +++ b/routes/api.js @@ -197,6 +197,7 @@ var run = function(req, res, runner, count){ console.log('run', runner); if(!runner || count > 3){ + console.log('no runner'); res.status(503); return res.json({error: 'No runners, try again soon.'}); } From 00bca4ff34f6f5e5b717a6c90779fec9581b8c3c Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:21:28 -0400 Subject: [PATCH 165/175] stuff --- routes/api.js | 2 +- testAPI.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 90ccbb4..916ee95 100644 --- a/routes/api.js +++ b/routes/api.js @@ -196,7 +196,7 @@ var run = function(req, res, runner, count){ }; console.log('run', runner); - if(!runner || count > 3){ + if(!runner){ console.log('no runner'); res.status(503); return res.json({error: 'No runners, try again soon.'}); diff --git a/testAPI.py b/testAPI.py index c407b8a..0832361 100644 --- a/testAPI.py +++ b/testAPI.py @@ -11,7 +11,7 @@ def testAPI(times=100, sleep=2): data={'code': 'pwd'} ) if res.status_code != 200: errors += 1 - print(i, res.status_code) + print(i, res.status_code, r.content) except: print('caught error') errors += 1 From 9c6903375647f21725f535b437e001cce4ff496a Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:41:12 -0400 Subject: [PATCH 166/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 916ee95..05e941a 100644 --- a/routes/api.js +++ b/routes/api.js @@ -196,7 +196,7 @@ var run = function(req, res, runner, count){ }; console.log('run', runner); - if(!runner){ + if(runner == null){ console.log('no runner'); res.status(503); return res.json({error: 'No runners, try again soon.'}); @@ -227,7 +227,7 @@ var getAvailrunner = function(runner){ console.log('getAvailrunner, none found', runner); if(runner) return runner; console.log('no..') - return false; + return null; }; setTimeout(function(){ From 2a21be1825596ec5468ce60511cf468ac9a5c3c2 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:47:32 -0400 Subject: [PATCH 167/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 05e941a..6a5c554 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,7 +42,7 @@ var workers = (function(){ workers.create = function(){ return doapi.dropletCreate({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().replace('.',''), + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(0,4), image: '17575764' }, function(data){ data = JSON.parse(data); @@ -96,7 +96,7 @@ var workers = (function(){ stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().replace('.',''); + var name = 'crunner-'+(Math.random()*100).toString().slice(0,4); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner on', worker.name) From b808d23965d57d518c77c5902d9dd1f0a83b25d5 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:49:05 -0400 Subject: [PATCH 168/175] stuff --- testAPI.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testAPI.py b/testAPI.py index 0832361..e507afd 100644 --- a/testAPI.py +++ b/testAPI.py @@ -11,9 +11,9 @@ def testAPI(times=100, sleep=2): data={'code': 'pwd'} ) if res.status_code != 200: errors += 1 - print(i, res.status_code, r.content) + print(i, res.status_code, res.content) except: print('caught error') errors += 1 time.sleep(sleep) - print('errors ', errors, (errors/times)*100) \ No newline at end of file + print('errors ', errors, (errors/times)*100) From 11c7b1907d89a5729e47edeadab19dd1be9cb4f5 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:52:58 -0400 Subject: [PATCH 169/175] stuff --- routes/api.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/routes/api.js b/routes/api.js index 6a5c554..34f880f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -185,6 +185,13 @@ var lxcTimeout = function(runner, time){ var run = function(req, res, runner, count){ count = count || 0; console.log('run start', count, runner); + + if(!runner){ + console.log('no runner'); + res.status(503); + return res.json({error: 'No runners, try again soon.'}); + } + var httpOptions = { url: 'http://' + runner.worker.ip, headers: { @@ -196,11 +203,6 @@ var run = function(req, res, runner, count){ }; console.log('run', runner); - if(runner == null){ - console.log('no runner'); - res.status(503); - return res.json({error: 'No runners, try again soon.'}); - } return request.post(httpOptions, function(error, response, body){ // console.log('runner response:', arguments) @@ -217,16 +219,12 @@ var run = function(req, res, runner, count){ var getAvailrunner = function(runner){ for(let worker of workers){ - console.log('checking ', worker.name, ' with ', worker.availrunners.length, ' free workers'); if(worker.availrunners.length === 0) continue; if(runner && runner.worker.index <= worker.index) break; if(runner) runnerFree(runner); - console.log('getAvailrunner while loop', runner); return worker.getRunner(); } - console.log('getAvailrunner, none found', runner); if(runner) return runner; - console.log('no..') return null; }; @@ -312,7 +310,6 @@ router.get('/liststuff', function(req, res, next){ router.post('/run/:ip?', function doRun(req, res, next){ console.log('hit runner route'); var runner = getAvailrunner(label2runner[req.params.ip]); - console.log('route ', runner); return run(req, res, runner); }); From 93bf4b2577f53d0ec6cf69fef36436f89bf26913 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:58:32 -0400 Subject: [PATCH 170/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 34f880f..7541b9b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,7 +42,7 @@ var workers = (function(){ workers.create = function(){ return doapi.dropletCreate({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(0,4), + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-3,0), image: '17575764' }, function(data){ data = JSON.parse(data); @@ -96,7 +96,7 @@ var workers = (function(){ stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().slice(0,4); + var name = 'crunner-'+(Math.random()*100).toString().slice(-3,0); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner on', worker.name) From 0c464207156727df79c881a950a2ec91e5cf1064 Mon Sep 17 00:00:00 2001 From: william Date: Mon, 30 May 2016 23:59:13 -0400 Subject: [PATCH 171/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 7541b9b..34f880f 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,7 +42,7 @@ var workers = (function(){ workers.create = function(){ return doapi.dropletCreate({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-3,0), + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(0,4), image: '17575764' }, function(data){ data = JSON.parse(data); @@ -96,7 +96,7 @@ var workers = (function(){ stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().slice(-3,0); + var name = 'crunner-'+(Math.random()*100).toString().slice(0,4); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner on', worker.name) From d204f29526f66292fbc36b5fa2cfccf3f950f621 Mon Sep 17 00:00:00 2001 From: william Date: Tue, 31 May 2016 00:06:18 -0400 Subject: [PATCH 172/175] stuff --- routes/api.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/routes/api.js b/routes/api.js index 34f880f..a7fa601 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,7 +42,7 @@ var workers = (function(){ workers.create = function(){ return doapi.dropletCreate({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(0,4), + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-3,-0), image: '17575764' }, function(data){ data = JSON.parse(data); @@ -96,7 +96,7 @@ var workers = (function(){ stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().slice(0,4); + var name = 'crunner-'+(Math.random()*100).toString().slice(-3,-0); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner on', worker.name) @@ -182,6 +182,17 @@ var lxcTimeout = function(runner, time){ }, time); }; +var getAvailrunner = function(runner){ + for(let worker of workers){ + if(worker.availrunners.length === 0) continue; + if(runner && runner.worker.index <= worker.index) break; + if(runner) runnerFree(runner); + return worker.getRunner(); + } + if(runner) return runner; + return null; +}; + var run = function(req, res, runner, count){ count = count || 0; console.log('run start', count, runner); @@ -217,17 +228,6 @@ var run = function(req, res, runner, count){ }); }; -var getAvailrunner = function(runner){ - for(let worker of workers){ - if(worker.availrunners.length === 0) continue; - if(runner && runner.worker.index <= worker.index) break; - if(runner) runnerFree(runner); - return worker.getRunner(); - } - if(runner) return runner; - return null; -}; - setTimeout(function(){ console.log('Starting balance checking in 30 seconds') setInterval(workers.checkBalance, 15000); From d90e0aaa25c96db137871a27774c93ead901955e Mon Sep 17 00:00:00 2001 From: william Date: Tue, 31 May 2016 00:07:48 -0400 Subject: [PATCH 173/175] stuff --- routes/api.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index a7fa601..ad07c6d 100644 --- a/routes/api.js +++ b/routes/api.js @@ -42,7 +42,7 @@ var workers = (function(){ workers.create = function(){ return doapi.dropletCreate({ - name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-3,-0), + name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4), image: '17575764' }, function(data){ data = JSON.parse(data); @@ -96,7 +96,7 @@ var workers = (function(){ stopPercent = stopPercent || 80; ramPercentUsed(worker.ip, function(usedMemPercent){ if(usedMemPercent < stopPercent ){ - var name = 'crunner-'+(Math.random()*100).toString().slice(-3,-0); + var name = 'crunner-'+(Math.random()*100).toString().slice(-4); return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){ if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0); console.log('started runner on', worker.name) From e9bd665cc1a65c1d71742f279579224e15e276ac Mon Sep 17 00:00:00 2001 From: william Date: Tue, 31 May 2016 00:23:10 -0400 Subject: [PATCH 174/175] stuff --- routes/api.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index ad07c6d..955eda9 100644 --- a/routes/api.js +++ b/routes/api.js @@ -17,7 +17,11 @@ var isCheckingWorkers = false; var workers = (function(){ var workers = []; + + workers.currentCreatingMax = 2; + workers.currentCreating = 0; + workers.checkDroplet = function(id, time){ time = time || 10000; @@ -29,6 +33,7 @@ var workers = (function(){ return setTimeout(function(worker){ console.log('Ready to start runners!'); workers.startRunners(workers.makeWorkerObj(worker), true); + workers.currentCreating--; }, 20000, worker); }else{ console.log('Worker not ready, check again in ', time, 'MS'); @@ -41,12 +46,13 @@ var workers = (function(){ }; workers.create = function(){ + if(workers.currentCreating > workers.currentCreatingMax ) return doapi.dropletCreate({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4), image: '17575764' }, function(data){ data = JSON.parse(data); - + workers.currentCreating++; setTimeout(function(dopletNewID){ return workers.checkDroplet(dopletNewID); }, 70000, data.droplet.id); From de4ce8553ed362508382fd32e8ac992703df8168 Mon Sep 17 00:00:00 2001 From: william Date: Tue, 31 May 2016 00:28:31 -0400 Subject: [PATCH 175/175] stuff --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 955eda9..3950efd 100644 --- a/routes/api.js +++ b/routes/api.js @@ -46,7 +46,7 @@ var workers = (function(){ }; workers.create = function(){ - if(workers.currentCreating > workers.currentCreatingMax ) + if(workers.currentCreating > workers.currentCreatingMax ) return false; return doapi.dropletCreate({ name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4), image: '17575764'