From 14a628a8ff837aa6c1b5244be20d6018f03dce0c Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 9 Oct 2017 00:26:58 -0400 Subject: [PATCH 1/7] simpler --- routes/api.js | 19 +++++++++++++++---- testAPI.js | 14 ++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/routes/api.js b/routes/api.js index 95a26a6..6d4ac12 100644 --- a/routes/api.js +++ b/routes/api.js @@ -215,9 +215,10 @@ var workers = (function(){ workers.checkForZombies(); // if there are workers being created, stop scale up and down check - if(workers.currentCreating) return ; + if(workers.currentCreating+workers.length < workers.min) 'do nothing'; + else if(workers.currentCreating) + return ; - // scale up and down check // hold amount of workers with no used runners var lastMinAval = 0; @@ -234,13 +235,23 @@ var workers = (function(){ if(lastMinAval > workers.settings.minAvail){ // Remove workers if there are more then the settings states - console.log('Last 3 runners not used, killing last runner', workers.length); + console.log( + 'Last 3 runners not used, killing last runner', + 'lastMinAval:', lastMinAval, + 'minAvail:', workers.settings.minAvail, + 'workers:', workers.length + ); return workers.destroy(); } else if(lastMinAval < workers.settings.minAvail){ // creates workers if the settings file demands it - console.log('last 3 workers have no free runners, starting droplet'); + console.log( + 'last 3 workers have no free runners, starting droplet', + 'lastMinAval:', lastMinAval, + 'minAvail:', workers.settings.minAvail, + 'workers:', workers.length + ); return workers.create(); } diff --git a/testAPI.js b/testAPI.js index 1c7beec..501fd5f 100644 --- a/testAPI.js +++ b/testAPI.js @@ -1,16 +1,18 @@ const request = require('request'); -var till = 15; +var till = 5; var completed = 0; var errors = 0; for(let i=0; i Date: Mon, 9 Oct 2017 00:28:29 -0400 Subject: [PATCH 2/7] simpler --- routes/api.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/routes/api.js b/routes/api.js index 6d4ac12..c71d815 100644 --- a/routes/api.js +++ b/routes/api.js @@ -215,10 +215,9 @@ var workers = (function(){ workers.checkForZombies(); // if there are workers being created, stop scale up and down check - if(workers.currentCreating+workers.length < workers.min) 'do nothing'; + if(workers.currentCreating+workers.length < workers.min) null; else if(workers.currentCreating) - return ; - + return console.log('killing balance, worker is being created.'); // hold amount of workers with no used runners var lastMinAval = 0; @@ -236,7 +235,7 @@ var workers = (function(){ if(lastMinAval > workers.settings.minAvail){ // Remove workers if there are more then the settings states console.log( - 'Last 3 runners not used, killing last runner', + `Last ${workers.settings.minAvail} workers not used, killing last worker`, 'lastMinAval:', lastMinAval, 'minAvail:', workers.settings.minAvail, 'workers:', workers.length @@ -247,7 +246,7 @@ var workers = (function(){ } else if(lastMinAval < workers.settings.minAvail){ // creates workers if the settings file demands it console.log( - 'last 3 workers have no free runners, starting droplet', + 'last 3 workers have no free runners, starting worker', 'lastMinAval:', lastMinAval, 'minAvail:', workers.settings.minAvail, 'workers:', workers.length From 5711070d92e0ad84b2bfa4f67ec07ddd06441375 Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 9 Oct 2017 00:32:24 -0400 Subject: [PATCH 3/7] simpler --- routes/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.js b/routes/api.js index c71d815..e339f7b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -214,6 +214,7 @@ var workers = (function(){ workers.checkForZombies(); + console.log(workers.currentCreating+workers.length, workers.currentCreating+workers.length < workers.min) // if there are workers being created, stop scale up and down check if(workers.currentCreating+workers.length < workers.min) null; else if(workers.currentCreating) From 128320f80e58264f85231d1b725878b0a4a4a40f Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 9 Oct 2017 00:39:02 -0400 Subject: [PATCH 4/7] fixes --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index e339f7b..a813971 100644 --- a/routes/api.js +++ b/routes/api.js @@ -216,7 +216,7 @@ var workers = (function(){ console.log(workers.currentCreating+workers.length, workers.currentCreating+workers.length < workers.min) // if there are workers being created, stop scale up and down check - if(workers.currentCreating+workers.length < workers.min) null; + if(workers.currentCreating+workers.length < workers.settings.min) null; else if(workers.currentCreating) return console.log('killing balance, worker is being created.'); From cfd6c373ad2e96703c9975023584612d08e3ff6b Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 9 Oct 2017 01:51:34 -0400 Subject: [PATCH 5/7] fixes --- routes/api.js | 53 ++++++++++++++++++++--------------- testAPI.js | 77 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 82 insertions(+), 48 deletions(-) diff --git a/routes/api.js b/routes/api.js index a813971..3b11390 100644 --- a/routes/api.js +++ b/routes/api.js @@ -64,10 +64,10 @@ var workers = (function(){ args.onStart = function(){}; }, onDone: function(args){ - console.log("done with runners on", worker.name); + console.log("Seeded runners on", worker.name); + workers.currentCreating--; } }); - workers.currentCreating--; } }); @@ -112,7 +112,7 @@ var workers = (function(){ var worker = worker || workers.pop(); return doapi.dropletDestroy(worker.id, function(body) { - console.log('body of destroy', body); + console.log('Deleted worker', worker.name); }); }; @@ -120,24 +120,26 @@ var workers = (function(){ // Delete works that with tag = tag || tagPrefix + workers.settings.version; - var currentIDs = workers.__workersId(); + let currentIDs = workers.__workersId(); - var deleteDroplets = function(droplets){ + let deleteDroplets = function(droplets){ if(droplets.length === 0) return true; - var droplet = droplets.pop(); + let droplet = droplets.pop(); if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets); doapi.dropletDestroy(droplet.id, function(body){ setTimeout(deleteDroplets, 1000, droplets); + if(!droplets.length) console.log(`Finished deleting workers tagged ${tag}.`); }); } doapi.dropletsByTag(tag, function(data){ data = JSON.parse(data); - console.log('current worker ids', currentIDs); - console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){ - return item.name+' | '+item.id; - })); + console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`, + data['droplets'].map(function(item){ + return item.name+' | '+item.id; + }) + ); deleteDroplets(data['droplets']); }); @@ -148,7 +150,7 @@ var workers = (function(){ // dont make runners on out dated workers if(!args.worker || workers.settings.image > args.worker.image.id){ - console.log('blocked outdated worker', workers.settings.image, args.worker.image.id) + console.log(`Blocked outdated worker(${args.worker.image.id}), current image ${workers.settings.image}.`) return ; } @@ -194,13 +196,13 @@ var workers = (function(){ let zombies = 0; for(let worker of workers){ - console.log("checking", worker.name, "if zombie"); + console.log(`Checking if ${worker.name} is a zombie worker.`); // if a runner has no available runners and no used runners, its a // zombie. This should happen when a newer image ID has been added // and old workers slowly lose there usefulness. if(worker.availrunners.length === 0 && worker.usedrunners === 0){ workers.splice(workers.indexOf(worker), 1); - console.log('found zombie worker, destroying'); + console.log(`Zombie! Worker ${worker.name}, destroying.`); workers.destroy(worker); zombies++; } @@ -210,15 +212,14 @@ var workers = (function(){ }; workers.checkBalance = function(){ - console.log('checking balance'); + console.log(`Checking balance.`); workers.checkForZombies(); - console.log(workers.currentCreating+workers.length, workers.currentCreating+workers.length < workers.min) // if there are workers being created, stop scale up and down check if(workers.currentCreating+workers.length < workers.settings.min) null; else if(workers.currentCreating) - return console.log('killing balance, worker is being created.'); + return console.log(`Killing balance, workers are being created.`); // hold amount of workers with no used runners var lastMinAval = 0; @@ -323,16 +324,16 @@ var getAvailrunner = function(runner){ var run = function(req, res, runner, count){ count = count || 0; - console.log('run start', count); + console.log(`Runner starting attempt ${count}.`); if(!runner){ - console.log('no runner'); + console.log(`No runner available!`); res.status(503); return res.json({error: 'No runners, try again soon.'}); } if(count > 2){ - console.log('to many reties on runner'); + console.log(`Runner attempt failed, to many requests!`); return res.status(400).json({error: 'Runner restarted to many times'}); } @@ -366,10 +367,9 @@ var run = function(req, res, runner, count){ }); }; +console.log('========STARTING===========') setInterval(workers.checkBalance, 15000); - workers.destroyByTag(); -workers.checkBalance(); router.get('/stop/:name', function(req, res, next){ @@ -458,7 +458,14 @@ router.post('/updateID', function(req, res, next){ router.get('/liststuff', function(req, res, next){ var obj = util.inspect(workers, {depth: 4}); - res.send("

Workers

"+obj+"

label2runner

"+util.inspect(label2runner)+'

DO calls

'+doapi.calls); + res.send(` +

Workers

+ 
${obj}
+

label2runner

+ 
${util.inspect(label2runner)}
+

DO calls

+ ${doapi.calls} + `); }); router.get('/ping/:runner', function(req, res, next){ @@ -468,7 +475,7 @@ router.get('/ping/:runner', function(req, res, next){ }); router.post('/run/:runner?', function (req, res, next){ - console.log('hit runner route'); + console.log(`Request runner route!`); var runner = getAvailrunner(label2runner[req.params.runner]); return run(req, res, runner); }); diff --git a/testAPI.js b/testAPI.js index 501fd5f..3131701 100644 --- a/testAPI.js +++ b/testAPI.js @@ -1,32 +1,59 @@ const request = require('request'); -var till = 5; -var completed = 0; -var errors = 0; +var callRunner = (function(){ + let started = 0; + let completed = 0; + let errors = 0; + let noRunner = 0; -for(let i=0; i Date: Mon, 9 Oct 2017 02:06:37 -0400 Subject: [PATCH 6/7] fixes --- routes/api.js | 1 + testAPI.js | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/routes/api.js b/routes/api.js index 3b11390..6a76aba 100644 --- a/routes/api.js +++ b/routes/api.js @@ -307,6 +307,7 @@ var runnerFree = function(runner){ } delete label2runner[runner.label]; + console.log(`Runner freed ${runner.label}.`, worker); workers.startRunners({worker: runner.worker}); }; diff --git a/testAPI.js b/testAPI.js index 3131701..d2b3f21 100644 --- a/testAPI.js +++ b/testAPI.js @@ -52,8 +52,10 @@ sleep(${sleepTime}) let __do = function(till){ + if(!till) return ; + callRunner(String(Math.random())[3]); - setTimeout(__do, 1000, --till); + setTimeout(__do, 1500, --till); }; -__do(50) \ No newline at end of file +__do(30) \ No newline at end of file From 127a2caac157db16e4615a02c02a121a3147a405 Mon Sep 17 00:00:00 2001 From: William Mantly Date: Mon, 9 Oct 2017 02:16:00 -0400 Subject: [PATCH 7/7] fixes --- routes/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index 6a76aba..bf929de 100644 --- a/routes/api.js +++ b/routes/api.js @@ -307,7 +307,7 @@ var runnerFree = function(runner){ } delete label2runner[runner.label]; - console.log(`Runner freed ${runner.label}.`, worker); + console.log(`Runner freed ${runner.label}.`, runner.worker); workers.startRunners({worker: runner.worker}); };