diff --git a/doapi.js b/doapi.js index a91177d..47f41ad 100644 --- a/doapi.js +++ b/doapi.js @@ -81,26 +81,29 @@ api = function(key){ } this.dropletToActive = function(args){ - var doapi = this; + args.__doapi = this; // hold the DO api in the agrs scope args.onCreated = args.onCreate || function(){}; this.dropletCreate(args, function(data){ data = JSON.parse(data); args.onCreate(data, args); - setTimeout(function check(id, args, doapi){ + + // check if the server is ready, giving time to allow + // digital ocean to do its thing + setTimeout(function check(id, args){ time = args.time || 10000; - doapi.dropletInfo(id, function (data){ + args.__doapi.dropletInfo(id, function (data){ var droplet = JSON.parse(data)['droplet']; if(droplet.status == 'active'){ return args.onActive(droplet, args); }else{ setTimeout(function(check, id){ - check(id, args, doapi); + check(id, args); }, time, check, droplet.id); } }); - }, 70000, data.droplet.id, args, doapi); + }, 70000, data.droplet.id, args); }); }; diff --git a/package.json b/package.json index b1becda..46613a0 100644 --- a/package.json +++ b/package.json @@ -11,10 +11,11 @@ "debug": "~2.2.0", "ejs": "~2.3.3", "express": "~4.13.1", + "jsonfile": "^2.3.1", "morgan": "~1.6.1", - "serve-favicon": "~2.3.0", "node.extend": "~1.1.5", "redis": "~2.4.2", - "request": "~2.69.0" + "request": "~2.69.0", + "serve-favicon": "~2.3.0" } } diff --git a/routes/api.js b/routes/api.js index a2b8dc2..e3882ff 100644 --- a/routes/api.js +++ b/routes/api.js @@ -4,6 +4,7 @@ var express = require('express'); var router = express.Router(); var util = require('util'); var request = require('request'); +var jsonfile = require('jsonfile'); var lxc = require('../lxc'); var doapi = require('../doapi')(); @@ -18,12 +19,12 @@ var workers = (function(){ workers.create = function(){ if(workers.currentCreating > workers.settings.max ) return false; workers.currentCreating++; - return doapi.dropletToActive({ + doapi.dropletToActive({ name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4), image: workers.settings.image, size: workers.settings.size, onCreate: function(data){ - doapi.dropletSetTag('clw'+workers.settings.version, data.droplet.id); + doapi.dropletSetTag('clwV'+workers.settings.version, data.droplet.id); }, onActive: function(worker, args){ workers.startRunners({ @@ -41,7 +42,7 @@ var workers = (function(){ onDone: function(args){ console.log("done with workers"); } - }), + }); workers.currentCreating--; } }); @@ -64,15 +65,10 @@ var workers = (function(){ var runner = this.availrunners.pop(); this.usedrunner++; runnerTimeout(runner); - return runner; } - worker.freeAll = function(){ - this.availrunners.forEach(runnerFree); - } - return worker; }; @@ -91,7 +87,7 @@ var workers = (function(){ }; workers.destroyOld = function(tag){ - tag = tag || 'clw'+workers.settings.version; + tag = tag || 'clwV'+workers.settings.version; var currentIDs = workers.__workersId(); var deleteDroplets = function(droplets){ if(droplets.length === 0) return true; @@ -116,13 +112,17 @@ var workers = (function(){ workers.startRunners = function(args){ // console.log('starting runners on', args.worker.name, args.worker.ip) - args.stopPercent = args.stopPercent || 80; + if(!args.worker || workers.settings.image > args.worker.image.id){ + console.log('blocked outdated worker', workers.settings.image, args.worker.image.id) + return ; + } + args.stopPercent = args.stopPercent || 20; args.onStart = args.onStart || function(){}; args.onDone = args.onDone || function(){}; ramPercentUsed(args.worker.ip, function(usedMemPercent){ if(usedMemPercent > args.stopPercent ){ - console.log('using', String(usedMemPercent), + console.log('using', String(usedMemPercent).trim(), 'percent memory, stopping runner creation!', args.worker.availrunners.length, 'created on ', args.worker.name ); @@ -160,9 +160,18 @@ var workers = (function(){ return ; } + for(let worker of workers){ + console.log("checking", worker.name, "if zombie") + if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + workers.splice(workers.indexOf(worker), 1) + console.log('found zombie worker, destroying'); + workers.destroy(worker); + } + } + if(workers.settings.minAvail > 0) return ; - var lastMinAval + var lastMinAval = 0; for(let worker of workers.slice(-wokers.settings.minAvail)){ if(worker.usedrunner !== 0){ console.log('last 3 workers have no free runners, starting droplet'); @@ -177,19 +186,20 @@ var workers = (function(){ return workers.destroy(); } - for(let worker of workers){ - if(worker.availrunners.length === 0 && worker.usedrunner === 0){ - console.log('found zombie worker, destroying') - workers.destroy(worker); - } - } + }; + workers.settingsSave = function(){ + jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) { + console.error(err) + }); }; - workers.concat = function(newWorkers){ - newWorkers.forEach(workers.push) - } + workers.add = function(newWorkers){ + newWorkers.forEach(function(worker){ + workers.push(worker); + }); + }; - doapi.tagCreate('clw'+workers.settings.version) + doapi.tagCreate('clwV'+workers.settings.version); return workers; })(); @@ -306,38 +316,62 @@ router.get('/destroyOld', function(req, res, next) { }); router.post('/updateID', function(req, res, next){ - var newWorkers = []; - var newWorkersTarget = workers.length; - var newWokerImage = req.query.image; - for(var i = 0; i= args.newWorkers.target){ + console.log('upgrade complete!') + workers.settings.image = args.newWorkers.image; + workers.settings.size = args.newWorkers.size; - workers.concat(newWorkers); + workers.forEach(function(worker){ + worker.availrunners.forEach(function(runner){ + lxc.stop(runner.name, runner.worker.ip); + }); + worker.availrunners = []; + }); + + workers.add(args.newWorkers.workers); + workers.settingsSave(); + workers.checkBalance(); } } - }) + + }); } }); } diff --git a/routes/workers.json b/routes/workers.json index fea18f4..231c92a 100644 --- a/routes/workers.json +++ b/routes/workers.json @@ -1 +1 @@ -{"image":"18473675","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file +{"image":"17641214","version":5,"lastSnapShotId":"0","size":"2gb","max":100,"min":1,"minAvail":1} \ No newline at end of file diff --git a/workers.json b/workers.json new file mode 100644 index 0000000..d92e9e3 --- /dev/null +++ b/workers.json @@ -0,0 +1,9 @@ +{ + "image": "18473675", + "version": 5, + "lastSnapShotId": "0", + "size": "2gb", + "max": 100, + "min": 1, + "minAvail": 1 +}