diff --git a/routes/api.js b/routes/api.js index 8253a3b..5ea6b6b 100644 --- a/routes/api.js +++ b/routes/api.js @@ -7,18 +7,39 @@ var request = require('request'); var jsonfile = require('jsonfile'); var lxc = require('../lxc'); var doapi = require('../doapi')(); +var settings = require('./workers.json'); +// mapping of current used runners for quick loop up based on runner label var label2runner = {}; var workers = (function(){ + // works array constructor. This will hold the works(order by creation) and all + // the methods interacting with the workers. + + // base array that will be the workers objects. var workers = []; - workers.settings = require('./workers.json'); + // persistent settings object + // .image is the currently used Digital Ocean snap shot ID + // .lastSnapShotId is the previous ID used Digital Ocean snap shot + // .version is the current worker version + // .size is the base Droplet size for worker creation + // .min is the minimum amount of workers that should exist + // .max is the maximum amount of works that ca exist + // .minAvail is the amount of empty workers there should be + workers.settings = settings; + + // How many droplets are currently in the process of being created. It takes + // about 3 minutes to create a worker. workers.currentCreating = 0; workers.create = function(){ + // manages the creation of a work from first call to all runners seeded + + // dont create more workers then the settings file allows if(workers.currentCreating > workers.settings.max ) return false; workers.currentCreating++; + doapi.dropletToActive({ name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4), image: workers.settings.image, @@ -51,20 +72,22 @@ var workers = (function(){ }; workers.makeWorkerObj = function(worker){ + // Create object for each worker. + worker.networks.v4.forEach(function(value){ worker[value.type+'IP'] = value.ip_address; }); worker.availrunners = []; worker.ip = worker.publicIP; - worker.usedrunner = 0; + worker.usedrunners = 0; worker.index = workers.length; worker.getRunner = function(){ if(this.availrunners.length === 0) return false; // console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length); var runner = this.availrunners.pop(); - this.usedrunner++; + this.usedrunners++; runnerTimeout(runner); return runner; @@ -74,6 +97,7 @@ var workers = (function(){ }; workers.__workersId = function(argument){ + // create array of all current worker Digital Ocean ID return workers.map(function(item){ return item.id; }); @@ -81,15 +105,21 @@ var workers = (function(){ }; workers.destroy = function(worker){ + // todo: If worker is passed, check for it in the workers array and + // remove it if found. + var worker = worker || workers.pop(); return doapi.dropletDestroy(worker.id, function(body) { console.log('body of destroy', body); }); }; - workers.destroyOld = function(tag){ - tag = tag || 'clwV'+workers.settings.version; + workers.destroyByTag = function(tag){ + // Delete works that with + + tag = tag || 'clwV' + workers.settings.version; var currentIDs = workers.__workersId(); + var deleteDroplets = function(droplets){ if(droplets.length === 0) return true; var droplet = droplets.pop(); @@ -113,10 +143,14 @@ var workers = (function(){ workers.startRunners = function(args){ // console.log('starting runners on', args.worker.name, args.worker.ip) + + // dont make runners on out dated workers if(!args.worker || workers.settings.image > args.worker.image.id){ console.log('blocked outdated worker', workers.settings.image, args.worker.image.id) return ; } + + // percent of used RAM to stop runner creation args.stopPercent = args.stopPercent || 80; args.onStart = args.onStart || function(){}; args.onDone = args.onDone || function(){}; @@ -152,43 +186,68 @@ var workers = (function(){ }); }; - workers.checkBalance = function(){ - console.log('checking balance'); - - if(workers.length < workers.settings.min){ - console.log('less then '+ workers.settings.min +' workers, starting a droplet'); - for(var i=workers.settings.min-workers.length; i--;) workers.create(); - return ; - } + workers.checkForZombies = function(){ + // check to make sure all works are used or usable. + + let zombies = 0; for(let worker of workers){ console.log("checking", worker.name, "if zombie"); - if(worker.availrunners.length === 0 && worker.usedrunner === 0){ + // if a runner has no available runners and no used runners, its a + // zombie. This should happen when a newer image ID has been added + // and old workers slowly lose there usefulness. + if(worker.availrunners.length === 0 && worker.usedrunners === 0){ workers.splice(workers.indexOf(worker), 1); console.log('found zombie worker, destroying'); workers.destroy(worker); + zombie++; } } + return zombie; + }; + + workers.checkBalance = function(){ + console.log('checking balance'); + + workers.checkForZombies(); + + // if there are workers being created, stop scale up and down check if(workers.currentCreating) return ; + // scale up and down check + + // hold amount of workers with no used runners var lastMinAval = 0; + + // check to make sure the `workers.settings.minAvail` have free runners for(let worker of workers.slice(-workers.settings.minAvail)){ - if(worker.usedrunner !== 0){ - console.log('last 3 workers have no free runners, starting droplet'); - return workers.create(); - } - if(workers.length > workers.settings.min && workers.usedrunner === 0 ){ + if(worker.usedrunners === 0){ lastMinAval++; + }else{ + // no need to keep counting, workers need to be created + break; } } - if(lastMinAval == workers.settings.minAvail){ + + if(lastMinAval > workers.settings.minAvail){ + // Remove workers if there are more then the settings states console.log('Last 3 runners not used, killing last runner', workers.length); + return workers.destroy(); + + }else(lastMinAval < workers.settings.minAvail){ + // creates workers if the settings file demands it + console.log('last 3 workers have no free runners, starting droplet'); + + return workers.create(); + } }; workers.settingsSave = function(){ + // save the live settings file to disk + jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) { console.error(err); }); @@ -200,12 +259,15 @@ var workers = (function(){ }); }; + // make sure Digital Ocean has a tag for the current worker version doapi.tagCreate('clwV'+workers.settings.version); + return workers; })(); var ramPercentUsed = function(ip, callback){ + // checks the percent of ram used on a worker. return lxc.exec( "python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"", @@ -226,7 +288,7 @@ var runnerTimeout = function(runner, time){ var runnerFree = function(runner){ lxc.stop(runner.name, runner.worker.ip); - runner.worker.usedrunner--; + runner.worker.usedrunners--; if(runner.hasOwnProperty('timeout')){ clearTimeout(runner.timeout); } @@ -297,7 +359,7 @@ setTimeout(function(){ setInterval(workers.checkBalance, 15000); }, 600000); -workers.destroyOld(); +workers.destroyByTag(); workers.checkBalance(); @@ -311,8 +373,8 @@ router.get('/stop/:name', function(req, res, next){ } }); }); -router.get('/destroyOld', function(req, res, next) { - workers.destroyOld(); +router.get('/destroyByTag', function(req, res, next) { + workers.destroyByTag(); res.send('?'); }); @@ -328,7 +390,7 @@ router.post('/updateID', function(req, res, next){ }; doapi.tagCreate('clwV'+newWorkers.version); - workers.destroyOld('clwV'+newWorkers.version); + workers.destroyByTag('clwV'+newWorkers.version); for(var i=0; i