docs and fixe

routes/api.js

@@ -7,18 +7,39 @@ var request = require('request');
 var jsonfile = require('jsonfile');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
+var settings = require('./workers.json');
 
+// mapping of current used runners for quick loop up based on runner label
 var label2runner = {};
 
 var workers = (function(){
+	// works array constructor. This will hold the works(order by creation) and all
+	// the methods interacting with the workers.
+	
+	// base array that will be the workers objects.
 	var workers = [];
-	workers.settings = require('./workers.json');
 
+	// persistent settings object
+	// .image is the currently used Digital Ocean snap shot ID
+	// .lastSnapShotId is the previous ID used Digital Ocean snap shot
+	// .version is the current worker version
+	// .size is the base Droplet size for worker creation
+	// .min is the minimum amount of workers that should exist
+	// .max is the maximum amount of works that ca exist
+	// .minAvail is the amount of empty workers there should be
+	workers.settings = settings;
+
+	// How many droplets are currently in the process of being created. It takes
+	// about 3 minutes to create a worker.
 	workers.currentCreating = 0;
 
 	workers.create = function(){
+		// manages the creation of a work from first call to all runners seeded
+
+		// dont create more workers then the settings file allows
 		if(workers.currentCreating > workers.settings.max ) return false;
 		workers.currentCreating++;
+
 		doapi.dropletToActive({
 			name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4),
 			image: workers.settings.image,
@@ -51,20 +72,22 @@ var workers = (function(){
 	};
 
 	workers.makeWorkerObj = function(worker){
+		// Create object for each worker.
+
 		worker.networks.v4.forEach(function(value){
 			worker[value.type+'IP'] = value.ip_address;
 		});
 
 		worker.availrunners = [];
 		worker.ip = worker.publicIP;
-		worker.usedrunner = 0;
+		worker.usedrunners = 0;
 		worker.index = workers.length;
 
 		worker.getRunner = function(){
 			if(this.availrunners.length === 0) return false;
 			// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
 			var runner = this.availrunners.pop();
-			this.usedrunner++;
+			this.usedrunners++;
 			runnerTimeout(runner);
 						
 			return runner;
@@ -74,6 +97,7 @@ var workers = (function(){
 	};
 
 	workers.__workersId = function(argument){
+		// create array of all current worker Digital Ocean ID
 		return workers.map(function(item){
 			return item.id;
 		});
@@ -81,15 +105,21 @@ var workers = (function(){
 	};
 
 	workers.destroy = function(worker){
+		// todo: If worker is passed, check for it in the workers array and
+		// remove it if found.
+
 		var worker = worker || workers.pop();
 		return doapi.dropletDestroy(worker.id, function(body) {
 			console.log('body of destroy', body);
 		});
 	};
 
-	workers.destroyOld = function(tag){
+	workers.destroyByTag = function(tag){
-		tag = tag || 'clwV'+workers.settings.version;
+		// Delete works that with
+
+		tag = tag || 'clwV' + workers.settings.version;
 		var currentIDs = workers.__workersId();
+
 		var deleteDroplets = function(droplets){
 			if(droplets.length === 0) return true;
 			var droplet = droplets.pop();
@@ -113,10 +143,14 @@ var workers = (function(){
 
 	workers.startRunners = function(args){ 
 		// console.log('starting runners on', args.worker.name, args.worker.ip)
+
+		// dont make runners on out dated workers
 		if(!args.worker || workers.settings.image > args.worker.image.id){
 			console.log('blocked outdated worker', workers.settings.image, args.worker.image.id)
 			return ;
 		}
+
+		// percent of used RAM to stop runner creation
 		args.stopPercent = args.stopPercent || 80;
 		args.onStart = args.onStart || function(){};
 		args.onDone = args.onDone || function(){};
@@ -152,43 +186,68 @@ var workers = (function(){
 		});
 	};
 
-	workers.checkBalance = function(){
+	workers.checkForZombies = function(){
-		console.log('checking balance');
+		// check to make sure all works are used or usable.
-
+		
-		if(workers.length < workers.settings.min){
+		let zombies = 0;
-			console.log('less then '+ workers.settings.min +' workers, starting a droplet');
-			for(var i=workers.settings.min-workers.length; i--;) workers.create();
-			return ;
-		}
 
 		for(let worker of workers){
 			console.log("checking", worker.name, "if zombie");
-			if(worker.availrunners.length === 0 && worker.usedrunner === 0){
+			// if a runner has no available runners and no used runners, its a
+			// zombie. This should happen when a newer image ID has been added
+			// and old workers slowly lose there usefulness.
+			if(worker.availrunners.length === 0 && worker.usedrunners === 0){
 				workers.splice(workers.indexOf(worker), 1);
 				console.log('found zombie worker, destroying');
 				workers.destroy(worker);
+				zombie++;
 			}
 		}
 
+		return zombie;
+	};
+
+	workers.checkBalance = function(){
+		console.log('checking balance');
+
+		workers.checkForZombies();
+
+		// if there are workers being created, stop scale up and down check
 		if(workers.currentCreating) return ;
 
+		// scale up and down check
+
+		// hold amount of workers with no used runners
 		var lastMinAval = 0;
+
+		// check to make sure the `workers.settings.minAvail` have free runners
 		for(let worker of workers.slice(-workers.settings.minAvail)){
-			if(worker.usedrunner !== 0){
+			if(worker.usedrunners === 0){
-				console.log('last 3 workers have no free runners, starting droplet');
-				return workers.create();
-			}
-			if(workers.length > workers.settings.min && workers.usedrunner === 0 ){
 				lastMinAval++;
+			}else{
+				// no need to keep counting, workers need to be created
+				break;
 			}
 		}
-		if(lastMinAval == workers.settings.minAvail){
+
+		if(lastMinAval > workers.settings.minAvail){
+			// Remove workers if there are more then the settings states
 			console.log('Last 3 runners not used, killing last runner', workers.length);
+
 			return workers.destroy();
+
+		}else(lastMinAval < workers.settings.minAvail){
+			// creates workers if the settings file demands it
+			console.log('last 3 workers have no free runners, starting droplet');
+
+			return workers.create();
+
 		}
 
 	};
 	workers.settingsSave = function(){
+		// save the live settings file to disk
+
 		jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) {
 			console.error(err);
 		});
@@ -200,12 +259,15 @@ var workers = (function(){
 		});
 	};
 
+	// make sure Digital Ocean has a tag for the current worker version
 	doapi.tagCreate('clwV'+workers.settings.version);
+
 	return workers;
 
 })();
 
 var ramPercentUsed = function(ip, callback){
+	// checks the percent of ram used on a worker.
 
 	return lxc.exec(
 		"python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"",
@@ -226,7 +288,7 @@ var runnerTimeout = function(runner, time){
 
 var runnerFree = function(runner){
 	lxc.stop(runner.name, runner.worker.ip);
-	runner.worker.usedrunner--;
+	runner.worker.usedrunners--;
 	if(runner.hasOwnProperty('timeout')){
 		clearTimeout(runner.timeout);
 	}
@@ -297,7 +359,7 @@ setTimeout(function(){
 	setInterval(workers.checkBalance, 15000);
 }, 600000);
 
-workers.destroyOld();
+workers.destroyByTag();
 workers.checkBalance();
 
 
@@ -311,8 +373,8 @@ router.get('/stop/:name', function(req, res, next){
 		}
 	});
 });
-router.get('/destroyOld', function(req, res, next) {
+router.get('/destroyByTag', function(req, res, next) {
-	workers.destroyOld();
+	workers.destroyByTag();
 	res.send('?');
 });
 
@@ -328,7 +390,7 @@ router.post('/updateID', function(req, res, next){
 	};
 
 	doapi.tagCreate('clwV'+newWorkers.version);
-	workers.destroyOld('clwV'+newWorkers.version);
+	workers.destroyByTag('clwV'+newWorkers.version);
 
 	for(var i=0; i<newWorkers.target; i++){