From 2c92d0f23204b2f5f4d0b4e482afc3cd4bf2bff4 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 9 Oct 2017 16:46:56 -0400
Subject: [PATCH 01/42] moved workers definition to seperate file.

---
 routes/api.js             | 264 +-----------------------------------
 routes/workers_manager.js | 275 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 276 insertions(+), 263 deletions(-)
 create mode 100644 routes/workers_manager.js

diff --git a/routes/api.js b/routes/api.js
index bf929de..51f7c66 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -15,269 +15,7 @@ var label2runner = {};
 // 
 var tagPrefix = settings.tagPrefix || 'clwV';
 
-var workers = (function(){
-	// works array constructor. This will hold the works(order by creation) and all
-	// the methods interacting with the workers.
-	
-	// base array that will be the workers objects.
-	var workers = [];
-
-	// persistent settings object
-	// .image is the currently used Digital Ocean snap shot ID
-	// .lastSnapShotId is the previous ID used Digital Ocean snap shot
-	// .version is the current worker version
-	// .size is the base Droplet size for worker creation
-	// .min is the minimum amount of workers that should exist
-	// .max is the maximum amount of works that ca exist
-	// .minAvail is the amount of empty workers there should be
-	workers.settings = settings;
-
-	// How many droplets are currently in the process of being created. It takes
-	// about 3 minutes to create a worker.
-	workers.currentCreating = 0;
-
-	workers.create = function(){
-		// manages the creation of a work from first call to all runners seeded
-
-		// dont create more workers then the settings file allows
-		if(workers.currentCreating > workers.settings.max ) return false;
-		workers.currentCreating++;
-
-		doapi.dropletToActive({
-			name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4),
-			image: workers.settings.image,
-			size: workers.settings.size,
-			onCreate: function(data){
-				doapi.dropletSetTag(tagPrefix+workers.settings.version, data.droplet.id);
-			},
-			onActive: function(worker, args){
-				workers.startRunners({
-					worker: workers.makeWorkerObj(worker),
-					onStart: function(runner, args){
-						workers.push(args.worker);
-						doapi.domianAddRecord({
-							domain: "codeland.us",
-							type: "A",
-							name: "*."+worker.name+".workers",
-							data: worker.publicIP
-						});
-						args.onStart = function(){};
-					},
-					onDone: function(args){
-						console.log("Seeded runners on", worker.name);
-						workers.currentCreating--;
-					}
-				});
-			}
-		});
-
-	};
-
-	workers.makeWorkerObj = function(worker){
-		// Create object for each worker.
-
-		worker.networks.v4.forEach(function(value){
-			worker[value.type+'IP'] = value.ip_address;
-		});
-
-		worker.availrunners = [];
-		worker.ip = worker.publicIP;
-		worker.usedrunners = 0;
-		worker.index = workers.length;
-
-		worker.getRunner = function(){
-			if(this.availrunners.length === 0) return false;
-			// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
-			var runner = this.availrunners.pop();
-			this.usedrunners++;
-			runnerTimeout(runner);
-						
-			return runner;
-		}
-
-		return worker;
-	};
-
-	workers.__workersId = function(argument){
-		// create array of all current worker Digital Ocean ID
-		return workers.map(function(item){
-			return item.id;
-		});
-
-	};
-
-	workers.destroy = function(worker){
-		// todo: If worker is passed, check for it in the workers array and
-		// remove it if found.
-
-		var worker = worker || workers.pop();
-		return doapi.dropletDestroy(worker.id, function(body) {
-			console.log('Deleted worker', worker.name);
-		});
-	};
-
-	workers.destroyByTag = function(tag){
-		// Delete works that with
-
-		tag = tag || tagPrefix + workers.settings.version;
-		let currentIDs = workers.__workersId();
-
-		let deleteDroplets = function(droplets){
-			if(droplets.length === 0) return true;
-			let droplet = droplets.pop();
-			if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
-			
-			doapi.dropletDestroy(droplet.id, function(body){
-				setTimeout(deleteDroplets, 1000, droplets);
-				if(!droplets.length) console.log(`Finished deleting workers tagged ${tag}.`);
-			});
-		}
-
-		doapi.dropletsByTag(tag, function(data){
-			data = JSON.parse(data);
-			console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
-				data['droplets'].map(function(item){
-					return item.name+' | '+item.id;
-				})
-			);
-
-			deleteDroplets(data['droplets']);
-		});
-	};
-
-	workers.startRunners = function(args){ 
-		// console.log('starting runners on', args.worker.name, args.worker.ip)
-
-		// dont make runners on out dated workers
-		if(!args.worker || workers.settings.image > args.worker.image.id){
-			console.log(`Blocked outdated worker(${args.worker.image.id}), current image ${workers.settings.image}.`)
-			return ;
-		}
-
-		// percent of used RAM to stop runner creation
-		args.stopPercent = args.stopPercent || 80;
-		args.onStart = args.onStart || function(){};
-		args.onDone = args.onDone || function(){};
-
-		ramPercentUsed(args.worker.ip, function(usedMemPercent){
-			if(usedMemPercent > args.stopPercent ){
-				console.log('using', String(usedMemPercent).trim(), 
-					'percent memory, stopping runner creation!', args.worker.availrunners.length, 
-					'created on ', args.worker.name
-				);
-				args.onDone(args);
-				return ;
-			}
-
-			var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
-			// console.log('Free ram check passed!')
-			lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
-				if(!data.ip) return setTimeout(workers.startRunners, 0, args);
-				// console.log('started runner on', args.worker.name)
-
-				var runner = {
-					ip: data.ip,
-					name: name,
-					worker: args.worker,
-					label: args.worker.name + ':' + name
-				};
-				args.onStart(runner, args);
-
-				args.worker.availrunners.push(runner);
-
-				setTimeout(workers.startRunners, 0, args);
-			});
-		});
-	};
-
-	workers.checkForZombies = function(){
-		// check to make sure all works are used or usable.
-		
-		let zombies = 0;
-
-		for(let worker of workers){
-			console.log(`Checking if ${worker.name} is a zombie worker.`);
-			// if a runner has no available runners and no used runners, its a
-			// zombie. This should happen when a newer image ID has been added
-			// and old workers slowly lose there usefulness.
-			if(worker.availrunners.length === 0 && worker.usedrunners === 0){
-				workers.splice(workers.indexOf(worker), 1);
-				console.log(`Zombie! Worker ${worker.name}, destroying.`);
-				workers.destroy(worker);
-				zombies++;
-			}
-		}
-
-		return zombies;
-	};
-
-	workers.checkBalance = function(){
-		console.log(`Checking balance.`);
-
-		workers.checkForZombies();
-
-		// if there are workers being created, stop scale up and down check
-		if(workers.currentCreating+workers.length < workers.settings.min) null;
-		else if(workers.currentCreating)
-			return console.log(`Killing balance, workers are being created.`);
-
-		// hold amount of workers with no used runners
-		var lastMinAval = 0;
-
-		// check to make sure the `workers.settings.minAvail` have free runners
-		for(let worker of workers.slice(-workers.settings.minAvail)){
-			if(worker.usedrunners === 0){
-				lastMinAval++;
-			}else{
-				// no need to keep counting, workers need to be created
-				break;
-			}
-		}
-
-		if(lastMinAval > workers.settings.minAvail){
-			// Remove workers if there are more then the settings states
-			console.log(
-				`Last ${workers.settings.minAvail} workers not used, killing last worker`, 
-				'lastMinAval:', lastMinAval,
-				'minAvail:', workers.settings.minAvail,
-				'workers:', workers.length
-			);
-
-			return workers.destroy();
-
-		} else if(lastMinAval < workers.settings.minAvail){
-			// creates workers if the settings file demands it
-			console.log(
-				'last 3 workers have no free runners, starting worker',
-				'lastMinAval:', lastMinAval,
-				'minAvail:', workers.settings.minAvail,
-				'workers:', workers.length
-			);
-
-			return workers.create();
-		}
-
-	};
-	workers.settingsSave = function(){
-		// save the live settings file to disk
-
-		jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) {
-			console.error(err);
-		});
-	};
-
-	workers.add = function(newWorkers){
-		newWorkers.forEach(function(worker){
-			workers.push(worker);
-		});
-	};
-
-	// make sure Digital Ocean has a tag for the current worker version
-	doapi.tagCreate(tagPrefix+workers.settings.version);
-
-	return workers;
-
-})();
+var workers = require('./workers_manager.js'); 
 
 var ramPercentUsed = function(ip, callback){
 	// checks the percent of ram used on a worker.
diff --git a/routes/workers_manager.js b/routes/workers_manager.js
new file mode 100644
index 0000000..b314746
--- /dev/null
+++ b/routes/workers_manager.js
@@ -0,0 +1,275 @@
+'use strict';
+
+var jsonfile = require('jsonfile');
+var lxc = require('../lxc');
+var doapi = require('../doapi')();
+var settings = require('./workers.json');
+
+var tagPrefix = settings.tagPrefix || 'clwV';
+
+var workers = (function(){
+	// works array constructor. This will hold the works(order by creation) and all
+	// the methods interacting with the workers.
+	
+	// base array that will be the workers objects.
+	var workers = [];
+
+	// persistent settings object
+	// .image is the currently used Digital Ocean snap shot ID
+	// .lastSnapShotId is the previous ID used Digital Ocean snap shot
+	// .version is the current worker version
+	// .size is the base Droplet size for worker creation
+	// .min is the minimum amount of workers that should exist
+	// .max is the maximum amount of works that ca exist
+	// .minAvail is the amount of empty workers there should be
+	workers.settings = settings;
+
+	// How many droplets are currently in the process of being created. It takes
+	// about 3 minutes to create a worker.
+	workers.currentCreating = 0;
+
+	workers.create = function(){
+		// manages the creation of a work from first call to all runners seeded
+
+		// dont create more workers then the settings file allows
+		if(workers.currentCreating > workers.settings.max ) return false;
+		workers.currentCreating++;
+
+		doapi.dropletToActive({
+			name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4),
+			image: workers.settings.image,
+			size: workers.settings.size,
+			onCreate: function(data){
+				doapi.dropletSetTag(tagPrefix+workers.settings.version, data.droplet.id);
+			},
+			onActive: function(worker, args){
+				workers.startRunners({
+					worker: workers.makeWorkerObj(worker),
+					onStart: function(runner, args){
+						workers.push(args.worker);
+						doapi.domianAddRecord({
+							domain: "codeland.us",
+							type: "A",
+							name: "*."+worker.name+".workers",
+							data: worker.publicIP
+						});
+						args.onStart = function(){};
+					},
+					onDone: function(args){
+						console.log("Seeded runners on", worker.name);
+						workers.currentCreating--;
+					}
+				});
+			}
+		});
+
+	};
+
+	workers.makeWorkerObj = function(worker){
+		// Create object for each worker.
+
+		worker.networks.v4.forEach(function(value){
+			worker[value.type+'IP'] = value.ip_address;
+		});
+
+		worker.availrunners = [];
+		worker.ip = worker.publicIP;
+		worker.usedrunners = 0;
+		worker.index = workers.length;
+
+		worker.getRunner = function(){
+			if(this.availrunners.length === 0) return false;
+			// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
+			var runner = this.availrunners.pop();
+			this.usedrunners++;
+			runnerTimeout(runner);
+						
+			return runner;
+		}
+
+		return worker;
+	};
+
+	workers.__workersId = function(argument){
+		// create array of all current worker Digital Ocean ID
+		return workers.map(function(item){
+			return item.id;
+		});
+
+	};
+
+	workers.destroy = function(worker){
+		// todo: If worker is passed, check for it in the workers array and
+		// remove it if found.
+
+		var worker = worker || workers.pop();
+		return doapi.dropletDestroy(worker.id, function(body) {
+			console.log('Deleted worker', worker.name);
+		});
+	};
+
+	workers.destroyByTag = function(tag){
+		// Delete works that with
+
+		tag = tag || tagPrefix + workers.settings.version;
+		let currentIDs = workers.__workersId();
+
+		let deleteDroplets = function(droplets){
+			if(droplets.length === 0) return true;
+			let droplet = droplets.pop();
+			if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
+			
+			doapi.dropletDestroy(droplet.id, function(body){
+				setTimeout(deleteDroplets, 1000, droplets);
+				if(!droplets.length) console.log(`Finished deleting workers tagged ${tag}.`);
+			});
+		}
+
+		doapi.dropletsByTag(tag, function(data){
+			data = JSON.parse(data);
+			console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
+				data['droplets'].map(function(item){
+					return item.name+' | '+item.id;
+				})
+			);
+
+			deleteDroplets(data['droplets']);
+		});
+	};
+
+	workers.startRunners = function(args){ 
+		// console.log('starting runners on', args.worker.name, args.worker.ip)
+
+		// dont make runners on out dated workers
+		if(!args.worker || workers.settings.image > args.worker.image.id){
+			console.log(`Blocked outdated worker(${args.worker.image.id}), current image ${workers.settings.image}.`)
+			return ;
+		}
+
+		// percent of used RAM to stop runner creation
+		args.stopPercent = args.stopPercent || 80;
+		args.onStart = args.onStart || function(){};
+		args.onDone = args.onDone || function(){};
+
+		ramPercentUsed(args.worker.ip, function(usedMemPercent){
+			if(usedMemPercent > args.stopPercent ){
+				console.log('using', String(usedMemPercent).trim(), 
+					'percent memory, stopping runner creation!', args.worker.availrunners.length, 
+					'created on ', args.worker.name
+				);
+				args.onDone(args);
+				return ;
+			}
+
+			var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
+			// console.log('Free ram check passed!')
+			lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
+				if(!data.ip) return setTimeout(workers.startRunners, 0, args);
+				// console.log('started runner on', args.worker.name)
+
+				var runner = {
+					ip: data.ip,
+					name: name,
+					worker: args.worker,
+					label: args.worker.name + ':' + name
+				};
+				args.onStart(runner, args);
+
+				args.worker.availrunners.push(runner);
+
+				setTimeout(workers.startRunners, 0, args);
+			});
+		});
+	};
+
+	workers.checkForZombies = function(){
+		// check to make sure all works are used or usable.
+		
+		let zombies = 0;
+
+		for(let worker of workers){
+			console.log(`Checking if ${worker.name} is a zombie worker.`);
+			// if a runner has no available runners and no used runners, its a
+			// zombie. This should happen when a newer image ID has been added
+			// and old workers slowly lose there usefulness.
+			if(worker.availrunners.length === 0 && worker.usedrunners === 0){
+				workers.splice(workers.indexOf(worker), 1);
+				console.log(`Zombie! Worker ${worker.name}, destroying.`);
+				workers.destroy(worker);
+				zombies++;
+			}
+		}
+
+		return zombies;
+	};
+
+	workers.checkBalance = function(){
+		console.log(`Checking balance.`);
+
+		workers.checkForZombies();
+
+		// if there are workers being created, stop scale up and down check
+		if(workers.currentCreating+workers.length < workers.settings.min) null;
+		else if(workers.currentCreating)
+			return console.log(`Killing balance, workers are being created.`);
+
+		// hold amount of workers with no used runners
+		var lastMinAval = 0;
+
+		// check to make sure the `workers.settings.minAvail` have free runners
+		for(let worker of workers.slice(-workers.settings.minAvail)){
+			if(worker.usedrunners === 0){
+				lastMinAval++;
+			}else{
+				// no need to keep counting, workers need to be created
+				break;
+			}
+		}
+
+		if(lastMinAval > workers.settings.minAvail){
+			// Remove workers if there are more then the settings states
+			console.log(
+				`Last ${workers.settings.minAvail} workers not used, killing last worker`, 
+				'lastMinAval:', lastMinAval,
+				'minAvail:', workers.settings.minAvail,
+				'workers:', workers.length
+			);
+
+			return workers.destroy();
+
+		} else if(lastMinAval < workers.settings.minAvail){
+			// creates workers if the settings file demands it
+			console.log(
+				'last 3 workers have no free runners, starting worker',
+				'lastMinAval:', lastMinAval,
+				'minAvail:', workers.settings.minAvail,
+				'workers:', workers.length
+			);
+
+			return workers.create();
+		}
+
+	};
+	workers.settingsSave = function(){
+		// save the live settings file to disk
+
+		jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) {
+			console.error(err);
+		});
+	};
+
+	workers.add = function(newWorkers){
+		newWorkers.forEach(function(worker){
+			workers.push(worker);
+		});
+	};
+
+	// make sure Digital Ocean has a tag for the current worker version
+	doapi.tagCreate(tagPrefix+workers.settings.version);
+
+	return workers;
+
+})();
+
+
+module.exports = workers;
\ No newline at end of file
-- 
2.34.1


From a3107de459cb2c458ca19bd3b398d90d94b1ddea Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 9 Oct 2017 23:12:48 -0400
Subject: [PATCH 02/42] adding concept of Worker and Runner. Moved label2runner
 to Workers manager.

---
 routes/api.js                                 |  77 ++++++------
 .../{workers_manager.js => worker_manager.js} | 119 +++++++++++++++++-
 2 files changed, 150 insertions(+), 46 deletions(-)
 rename routes/{workers_manager.js => worker_manager.js} (76%)

diff --git a/routes/api.js b/routes/api.js
index 51f7c66..b9a23ac 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -15,51 +15,44 @@ var label2runner = {};
 // 
 var tagPrefix = settings.tagPrefix || 'clwV';
 
-var workers = require('./workers_manager.js'); 
+var workers = require('./worker_manager.js');
 
-var ramPercentUsed = function(ip, callback){
-	// checks the percent of ram used on a worker.
 
-	return lxc.exec(
-		"python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"",
-		ip,
-		callback
-	);
-};
 
-var runnerTimeout = function(runner, time){
-	time = time || 60000; // 1 minutes
+// var runnerTimeout = function(runner, time){
+// 	time = time || 60000; // 1 minutes
 
-	if(runner.hasOwnProperty('timeout')){
-		clearTimeout(runner.timeout);
-	}
+// 	if(runner.hasOwnProperty('timeout')){
+// 		clearTimeout(runner.timeout);
+// 	}
 
-	return runner.timeout = setTimeout(runnerFree, time, runner);
-};
+// 	return runner.timeout = setTimeout(runnerFree, time, runner);
+// };
 
-var runnerFree = function(runner){
-	lxc.stop(runner.name, runner.worker.ip);
-	runner.worker.usedrunners--;
-	if(runner.hasOwnProperty('timeout')){
-		clearTimeout(runner.timeout);
-	}
-	delete label2runner[runner.label];
+// var runnerFree = function(runner){
+// 	lxc.stop(runner.name, runner.worker.ip);
+// 	runner.worker.usedrunners--;
+// 	if(runner.hasOwnProperty('timeout')){
+// 		clearTimeout(runner.timeout);
+// 	}
+// 	delete label2runner[runner.label];
 
-	console.log(`Runner freed ${runner.label}.`, runner.worker);
-	workers.startRunners({worker: runner.worker});
-};
+// 	console.log(`Runner freed ${runner.label}.`, runner.worker);
+// 	workers.startRunners({worker: runner.worker});
+// };
 
-var getAvailrunner = function(runner){
-	for(let worker of workers){
-		if(worker.availrunners.length === 0) continue;
-		if(runner && runner.worker.index <= worker.index) break;
-		if(runner) runnerFree(runner);
+// var getAvailrunner = function(runner){
+// 	for(let worker of workers){
+// 		if(worker.availrunners.length === 0) continue;
+// 		if(runner && runner.worker.index <= worker.index) break;
+// 		// if(runner) runnerFree(runner);
+// 		if(runner) runner.free();
 
-		return worker.getRunner();
-	}
+// 		return worker.getRunner();
+// 	}
 
-	if(runner) return runner;
-};
+// 	if(runner) return runner;
+// };
 
 var run = function(req, res, runner, count){
 	count = count || 0;
@@ -88,12 +81,14 @@ var run = function(req, res, runner, count){
 
 	return request.post(httpOptions, function(error, response, body){
 		// console.log('runner response:', arguments)
-		if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count);
+		if(error || response.statusCode !== 200) return run(req, res, workers.getAvailableRunner(), ++count);
 		body = JSON.parse(body);
 
 		if(req.query.once){
 			res.json(body);
-			return runnerFree(runner, 0);
+			// 0 here does nothing
+			// return runnerFree(runner, 0);
+			return runner.free();
 		}
 
 		label2runner[runner.label] = runner;
@@ -102,7 +97,8 @@ var run = function(req, res, runner, count){
 		body['wname'] = runner.worker.name;
 		res.json(body);
 
-		runnerTimeout(runner);
+		// runnerTimeout(runner);
+		runner.setTimeout();
 	});
 };
 
@@ -209,13 +205,14 @@ router.get('/liststuff', function(req, res, next){
 
 router.get('/ping/:runner', function(req, res, next){
 	var runner = label2runner[req.params.runner];
-	runnerTimeout(runner);
+	// runnerTimeout(runner);
+	runner.setTimeout();
 	res.json({res:''});
 });
 
 router.post('/run/:runner?', function (req, res, next){
 	console.log(`Request runner route!`);
-	var runner = getAvailrunner(label2runner[req.params.runner]);
+	var runner = workers.getAvailrunner(workers.getRunner(req.params.runner));
 	return run(req, res, runner);
 });
 
diff --git a/routes/workers_manager.js b/routes/worker_manager.js
similarity index 76%
rename from routes/workers_manager.js
rename to routes/worker_manager.js
index b314746..df7d727 100644
--- a/routes/workers_manager.js
+++ b/routes/worker_manager.js
@@ -7,6 +7,88 @@ var settings = require('./workers.json');
 
 var tagPrefix = settings.tagPrefix || 'clwV';
 
+
+var ramPercentUsed = function(ip, callback){
+	// checks the percent of ram used on a worker.
+
+	return lxc.exec(
+		"python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"",
+		ip,
+		callback
+	);
+};
+
+var Runner = (function(){
+	var proto = {};
+
+	proto.create = function(config){
+		var runner = Object.create(proto);
+		Object.assign(runner, config);
+		return runner;
+	};
+
+	proto.free = function(){
+		var runner = this;
+		lxc.stop(runner.name, runner.worker.ip);
+		runner.worker.usedrunners--;
+		if(runner.hasOwnProperty('timeout')){
+			clearTimeout(runner.timeout);
+		};
+		
+
+		// TODO: FIX THIS
+		delete label2runner[runner.label];
+
+		console.log(`Runner freed ${runner.label}.`, runner.worker);
+		// Why does this need to run here?
+		workers.startRunners({worker: runner.worker});
+	};
+
+	proto.setTimeout = function(time){
+		time = time || 60000; // 1 minutes
+		var runner = this;
+		if(runner.hasOwnProperty('timeout')){
+			clearTimeout(runner.timeout);
+		}
+
+		return runner.timeout = setTimeout(function(){
+			runner.free();
+		}, time);
+	};
+
+})();
+
+
+var Worker = (function(){
+	var proto = {};
+
+	proto.create = function(config){
+		var worker = Object.create(proto);
+
+		worker.networks.v4.forEach(function(value){
+			worker[value.type+'IP'] = value.ip_address;
+		});
+
+		worker.availrunners = [];
+		worker.ip = worker.publicIP;
+		worker.usedrunners = 0;
+		worker.index = workers.length;
+
+		return worker;
+	};
+
+	proto.getRunner = function(){
+		if(this.availrunners.length === 0) return false;
+		// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
+		var runner = this.availrunners.pop();
+		this.usedrunners++;
+		runner.setTimeout();
+					
+		return runner;
+	};
+})();
+
+
 var workers = (function(){
 	// works array constructor. This will hold the works(order by creation) and all
 	// the methods interacting with the workers.
@@ -14,6 +96,32 @@ var workers = (function(){
 	// base array that will be the workers objects.
 	var workers = [];
 
+	var runnerMap = {};
+
+	workers.setRunner = function(runner){
+		runnerMap[runner.label] = runner;
+	};
+
+
+	workers.getRunner = function(label){
+		return runnerMap[runner.label];
+	};
+
+
+	workers.getAvailableRunner = function(runner){
+		for(let worker of workers){
+			if(worker.availrunners.length === 0) continue;
+			if(runner && runner.worker.index <= worker.index) break;
+			// if(runner) runnerFree(runner);
+			if(runner) runner.free();
+
+			return worker.getRunner();
+		}
+
+		if(runner) return runner;
+	};
+
+
 	// persistent settings object
 	// .image is the currently used Digital Ocean snap shot ID
 	// .lastSnapShotId is the previous ID used Digital Ocean snap shot
@@ -62,7 +170,6 @@ var workers = (function(){
 				});
 			}
 		});
-
 	};
 
 	workers.makeWorkerObj = function(worker){
@@ -85,7 +192,7 @@ var workers = (function(){
 			runnerTimeout(runner);
 						
 			return runner;
-		}
+		};
 
 		return worker;
 	};
@@ -95,7 +202,6 @@ var workers = (function(){
 		return workers.map(function(item){
 			return item.id;
 		});
-
 	};
 
 	workers.destroy = function(worker){
@@ -167,12 +273,13 @@ var workers = (function(){
 				if(!data.ip) return setTimeout(workers.startRunners, 0, args);
 				// console.log('started runner on', args.worker.name)
 
-				var runner = {
+				var runner = Runner.create({
 					ip: data.ip,
 					name: name,
 					worker: args.worker,
-					label: args.worker.name + ':' + name
-				};
+					label: args.worker.name + ':' + name,
+
+				});
 				args.onStart(runner, args);
 
 				args.worker.availrunners.push(runner);
-- 
2.34.1


From 6e0bf97142016c220bcd5749b220204ddf4c320b Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 00:48:03 -0400
Subject: [PATCH 03/42] removed '/updateID' route.

---
 routes/api.js            | 125 +++++----------------------------------
 routes/worker_manager.js |  77 ++++++++++--------------
 2 files changed, 44 insertions(+), 158 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index b9a23ac..a69366c 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -18,43 +18,7 @@ var tagPrefix = settings.tagPrefix || 'clwV';
 var workers = require('./worker_manager.js');
 
 
-
-// var runnerTimeout = function(runner, time){
-// 	time = time || 60000; // 1 minutes
-
-// 	if(runner.hasOwnProperty('timeout')){
-// 		clearTimeout(runner.timeout);
-// 	}
-
-// 	return runner.timeout = setTimeout(runnerFree, time, runner);
-// };
-
-// var runnerFree = function(runner){
-// 	lxc.stop(runner.name, runner.worker.ip);
-// 	runner.worker.usedrunners--;
-// 	if(runner.hasOwnProperty('timeout')){
-// 		clearTimeout(runner.timeout);
-// 	}
-// 	delete label2runner[runner.label];
-
-// 	console.log(`Runner freed ${runner.label}.`, runner.worker);
-// 	workers.startRunners({worker: runner.worker});
-// };
-
-// var getAvailrunner = function(runner){
-// 	for(let worker of workers){
-// 		if(worker.availrunners.length === 0) continue;
-// 		if(runner && runner.worker.index <= worker.index) break;
-// 		// if(runner) runnerFree(runner);
-// 		if(runner) runner.free();
-
-// 		return worker.getRunner();
-// 	}
-
-// 	if(runner) return runner;
-// };
-
-var run = function(req, res, runner, count){
+var attemptRun = function(req, res, runner, count){
 	count = count || 0;
 	console.log(`Runner starting attempt ${count}.`);
 
@@ -64,6 +28,7 @@ var run = function(req, res, runner, count){
 		return res.json({error: 'No runners, try again soon.'});
 	}
 
+	// TODO: Configurable
 	if(count > 2){
 		console.log(`Runner attempt failed, to many requests!`);
 		return res.status(400).json({error: 'Runner restarted to many times'});
@@ -81,7 +46,10 @@ var run = function(req, res, runner, count){
 
 	return request.post(httpOptions, function(error, response, body){
 		// console.log('runner response:', arguments)
-		if(error || response.statusCode !== 200) return run(req, res, workers.getAvailableRunner(), ++count);
+		if(error || response.statusCode !== 200) {
+			return attemptRun(req, res, workers.getAvailableRunner(), ++count);
+		}
+		
 		body = JSON.parse(body);
 
 		if(req.query.once){
@@ -91,7 +59,7 @@ var run = function(req, res, runner, count){
 			return runner.free();
 		}
 
-		label2runner[runner.label] = runner;
+		workers.setRunner(runner);
 		body['ip'] = runner.label;
 		body['rname'] = runner.name;
 		body['wname'] = runner.worker.name;
@@ -103,10 +71,11 @@ var run = function(req, res, runner, count){
 };
 
 console.log('========STARTING===========')
+// TODO: Make this a function
 setInterval(workers.checkBalance, 15000);
 workers.destroyByTag();
 
-
+// Why is this a GET?
 router.get('/stop/:name', function(req, res, next){
 	return lxc.stop(req.params.name, function(data){
 		console.log('stop', arguments);
@@ -117,79 +86,13 @@ router.get('/stop/:name', function(req, res, next){
 		}
 	});
 });
+
+// Why is this a GET?
 router.get('/destroyByTag', function(req, res, next) {
 	workers.destroyByTag();
 	res.send('?');
 });
 
-router.post('/updateID', function(req, res, next){
-	var newWorkers = {
-		workers: [],
-		image: req.query.image,
-		target: req.query.target || workers.length,
-		size: req.query.size || workers.settings.size,
-		version: workers.settings.version+1,
-		min: req.query.min || workers.settings,
-		minAvail: req.query.minAvail || workers.settings
-	};
-
-	doapi.tagCreate(tagPrefix+newWorkers.version);
-	workers.destroyByTag(tagPrefix+newWorkers.version);
-
-	for(var i=0; i<newWorkers.target; i++){
-
-		doapi.dropletToActive({
-			name: 'clw'+newWorkers.version+'-'+(Math.random()*100).toString().slice(-4),
-			newWorkers: newWorkers,
-			image: newWorkers.image,
-			size: newWorkers.size,
-			onCreate: function(data, args){
-				doapi.dropletSetTag(tagPrefix+args.newWorkers.version, data.droplet.id);
-			},
-			onActive: function(droplet, args){
-				workers.startRunners({
-					worker: workers.makeWorkerObj(droplet),
-					newWorkers: args.newWorkers,
-					onStart: function(runner, args){
-						args.newWorkers.workers.push(args.worker);
-						console.log('onStart', args.worker.name);
-						args.onStart = function(){};
-					},
-					onDone: function(args){
-						console.log('new workers:', args.newWorkers.workers.length);
-						doapi.domianAddRecord({
-							domain: "codeland.us",
-							type: "A",
-							name: "*."+args.worker.name+".workers",
-							data: args.worker.publicIP
-						});
-						
-						if(args.newWorkers.workers.length >= args.newWorkers.target){
-							console.log('upgrade complete!')
-							workers.settings.image = args.newWorkers.image;
-							workers.settings.size = args.newWorkers.size;
-							workers.settings.min = args.newWorkers.min;
-							workers.settings.minAvail = args.newWorkers.minAvail;
-
-							workers.forEach(function(worker){
-								worker.availrunners.forEach(function(runner){
-									lxc.stop(runner.name, runner.worker.ip);
-								});
-								worker.availrunners = [];
-							});
-
-							workers.add(args.newWorkers.workers);
-							workers.settingsSave();
-							workers.checkBalance();
-						}
-					}
-
-				});
-			}
-		});
-	}
-	res.json({status: "maybe?"});
-});
 
 router.get('/liststuff', function(req, res, next){
 	var obj = util.inspect(workers, {depth: 4});
@@ -197,14 +100,14 @@ router.get('/liststuff', function(req, res, next){
 		<h1>Workers</h1>
 		<pre>${obj}</pre>
 		<h1>label2runner</h1>
-		<pre>${util.inspect(label2runner)}</pre>
+		<pre>${util.inspect(workers.runnerMap)}</pre>
 		<h1>DO calls</h1>
 		${doapi.calls}
 	`);
 });
 
 router.get('/ping/:runner', function(req, res, next){
-	var runner = label2runner[req.params.runner];
+	var runner = workers.getRunner(req.params.runner);
 	// runnerTimeout(runner);
 	runner.setTimeout();
 	res.json({res:''});
@@ -213,7 +116,7 @@ router.get('/ping/:runner', function(req, res, next){
 router.post('/run/:runner?', function (req, res, next){
 	console.log(`Request runner route!`);
 	var runner = workers.getAvailrunner(workers.getRunner(req.params.runner));
-	return run(req, res, runner);
+	return attemptRun(req, res, runner);
 });
 
 module.exports = router;
diff --git a/routes/worker_manager.js b/routes/worker_manager.js
index df7d727..15acdce 100644
--- a/routes/worker_manager.js
+++ b/routes/worker_manager.js
@@ -8,16 +8,6 @@ var settings = require('./workers.json');
 var tagPrefix = settings.tagPrefix || 'clwV';
 
 
-var ramPercentUsed = function(ip, callback){
-	// checks the percent of ram used on a worker.
-
-	return lxc.exec(
-		"python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"",
-		ip,
-		callback
-	);
-};
-
 var Runner = (function(){
 	var proto = {};
 
@@ -86,6 +76,16 @@ var Worker = (function(){
 					
 		return runner;
 	};
+
+
+	proto.ramPercentUsed = function(callback){
+	// checks the percent of ram used on a worker.
+		return lxc.exec(
+			"python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"",
+			this.ip,
+			callback
+		);
+	};
 })();
 
 
@@ -96,7 +96,7 @@ var workers = (function(){
 	// base array that will be the workers objects.
 	var workers = [];
 
-	var runnerMap = {};
+	workers.runnerMap = {};
 
 	workers.setRunner = function(runner){
 		runnerMap[runner.label] = runner;
@@ -136,23 +136,28 @@ var workers = (function(){
 	// about 3 minutes to create a worker.
 	workers.currentCreating = 0;
 
-	workers.create = function(){
+	workers.create = function(config){
 		// manages the creation of a work from first call to all runners seeded
 
 		// dont create more workers then the settings file allows
 		if(workers.currentCreating > workers.settings.max ) return false;
 		workers.currentCreating++;
 
+		config = config || workers.settings;
+
 		doapi.dropletToActive({
-			name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4),
-			image: workers.settings.image,
-			size: workers.settings.size,
+			name: 'clw'+config.version+'-'+(Math.random()*100).toString().slice(-4),
+			image: config.image,
+			size: config.size,
 			onCreate: function(data){
-				doapi.dropletSetTag(tagPrefix+workers.settings.version, data.droplet.id);
+				doapi.dropletSetTag(
+					tagPrefix + config.version, 
+					data.droplet.id
+				);
 			},
-			onActive: function(worker, args){
+			onActive: function(data, args){
 				workers.startRunners({
-					worker: workers.makeWorkerObj(worker),
+					worker: Worker.create(data),
 					onStart: function(runner, args){
 						workers.push(args.worker);
 						doapi.domianAddRecord({
@@ -171,32 +176,7 @@ var workers = (function(){
 			}
 		});
 	};
-
-	workers.makeWorkerObj = function(worker){
-		// Create object for each worker.
-
-		worker.networks.v4.forEach(function(value){
-			worker[value.type+'IP'] = value.ip_address;
-		});
-
-		worker.availrunners = [];
-		worker.ip = worker.publicIP;
-		worker.usedrunners = 0;
-		worker.index = workers.length;
-
-		worker.getRunner = function(){
-			if(this.availrunners.length === 0) return false;
-			// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
-			var runner = this.availrunners.pop();
-			this.usedrunners++;
-			runnerTimeout(runner);
-						
-			return runner;
-		};
-
-		return worker;
-	};
-
+	
 	workers.__workersId = function(argument){
 		// create array of all current worker Digital Ocean ID
 		return workers.map(function(item){
@@ -257,7 +237,7 @@ var workers = (function(){
 		args.onStart = args.onStart || function(){};
 		args.onDone = args.onDone || function(){};
 
-		ramPercentUsed(args.worker.ip, function(usedMemPercent){
+		args.worker.ramPercentUsed(function(usedMemPercent){
 			if(usedMemPercent > args.stopPercent ){
 				console.log('using', String(usedMemPercent).trim(), 
 					'percent memory, stopping runner creation!', args.worker.availrunners.length, 
@@ -270,7 +250,9 @@ var workers = (function(){
 			var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
 			// console.log('Free ram check passed!')
 			lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
-				if(!data.ip) return setTimeout(workers.startRunners, 0, args);
+				if(!data.ip){
+					return setTimeout(workers.startRunners, 0, args);
+				}
 				// console.log('started runner on', args.worker.name)
 
 				var runner = Runner.create({
@@ -280,6 +262,7 @@ var workers = (function(){
 					label: args.worker.name + ':' + name,
 
 				});
+
 				args.onStart(runner, args);
 
 				args.worker.availrunners.push(runner);
@@ -372,7 +355,7 @@ var workers = (function(){
 	};
 
 	// make sure Digital Ocean has a tag for the current worker version
-	doapi.tagCreate(tagPrefix+workers.settings.version);
+	doapi.tagCreate(tagPrefix + workers.settings.version);
 
 	return workers;
 
-- 
2.34.1


From ca0b0ef1536a5d16aa916d346384008bc7aa2179 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 01:25:43 -0400
Subject: [PATCH 04/42] adding afterFreed hook

---
 routes/api.js            |  8 -----
 routes/worker_manager.js | 74 +++++++++++++++++++++-------------------
 2 files changed, 39 insertions(+), 43 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index a69366c..36546cd 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -4,16 +4,8 @@ var express = require('express');
 var router = express.Router();
 var util = require('util');
 var request = require('request');
-var jsonfile = require('jsonfile');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
-var settings = require('./workers.json');
-
-// mapping of current used runners for quick loop up based on runner label
-var label2runner = {};
-
-// 
-var tagPrefix = settings.tagPrefix || 'clwV';
 
 var workers = require('./worker_manager.js');
 
diff --git a/routes/worker_manager.js b/routes/worker_manager.js
index 15acdce..01f6a47 100644
--- a/routes/worker_manager.js
+++ b/routes/worker_manager.js
@@ -5,33 +5,26 @@ var lxc = require('../lxc');
 var doapi = require('../doapi')();
 var settings = require('./workers.json');
 
-var tagPrefix = settings.tagPrefix || 'clwV';
-
 
 var Runner = (function(){
 	var proto = {};
 
 	proto.create = function(config){
 		var runner = Object.create(proto);
+		var __empty = function(){};
 		Object.assign(runner, config);
+		runner.afterFreed = runner.afterFreed || __empty;
 		return runner;
 	};
 
-	proto.free = function(){
+	proto.free = function(callback){
 		var runner = this;
 		lxc.stop(runner.name, runner.worker.ip);
 		runner.worker.usedrunners--;
 		if(runner.hasOwnProperty('timeout')){
 			clearTimeout(runner.timeout);
 		};
-		
-
-		// TODO: FIX THIS
-		delete label2runner[runner.label];
-
-		console.log(`Runner freed ${runner.label}.`, runner.worker);
-		// Why does this need to run here?
-		workers.startRunners({worker: runner.worker});
+		runner.afterFreed();
 	};
 
 	proto.setTimeout = function(time){
@@ -96,8 +89,26 @@ var workers = (function(){
 	// base array that will be the workers objects.
 	var workers = [];
 
+	var tagPrefix = settings.tagPrefix || 'clwV';
+
 	workers.runnerMap = {};
 
+	// persistent settings object
+	// .image is the currently used Digital Ocean snap shot ID
+	// .lastSnapShotId is the previous ID used Digital Ocean snap shot
+	// .version is the current worker version
+	// .size is the base Droplet size for worker creation
+	// .min is the minimum amount of workers that should exist
+	// .max is the maximum amount of works that ca exist
+	// .minAvail is the amount of empty workers there should be
+	workers.settings = settings;
+
+	// How many droplets are currently in the process of being created. It takes
+	// about 3 minutes to create a worker.
+	workers.currentCreating = 0;
+
+
+	
 	workers.setRunner = function(runner){
 		runnerMap[runner.label] = runner;
 	};
@@ -121,21 +132,6 @@ var workers = (function(){
 		if(runner) return runner;
 	};
 
-
-	// persistent settings object
-	// .image is the currently used Digital Ocean snap shot ID
-	// .lastSnapShotId is the previous ID used Digital Ocean snap shot
-	// .version is the current worker version
-	// .size is the base Droplet size for worker creation
-	// .min is the minimum amount of workers that should exist
-	// .max is the maximum amount of works that ca exist
-	// .minAvail is the amount of empty workers there should be
-	workers.settings = settings;
-
-	// How many droplets are currently in the process of being created. It takes
-	// about 3 minutes to create a worker.
-	workers.currentCreating = 0;
-
 	workers.create = function(config){
 		// manages the creation of a work from first call to all runners seeded
 
@@ -146,7 +142,7 @@ var workers = (function(){
 		config = config || workers.settings;
 
 		doapi.dropletToActive({
-			name: 'clw'+config.version+'-'+(Math.random()*100).toString().slice(-4),
+			name: 'clw' + config.version + '-' + (Math.random()*100).toString().slice(-4),
 			image: config.image,
 			size: config.size,
 			onCreate: function(data){
@@ -163,7 +159,7 @@ var workers = (function(){
 						doapi.domianAddRecord({
 							domain: "codeland.us",
 							type: "A",
-							name: "*."+worker.name+".workers",
+							name: "*." + worker.name + ".workers",
 							data: worker.publicIP
 						});
 						args.onStart = function(){};
@@ -176,7 +172,7 @@ var workers = (function(){
 			}
 		});
 	};
-	
+
 	workers.__workersId = function(argument){
 		// create array of all current worker Digital Ocean ID
 		return workers.map(function(item){
@@ -239,7 +235,7 @@ var workers = (function(){
 
 		args.worker.ramPercentUsed(function(usedMemPercent){
 			if(usedMemPercent > args.stopPercent ){
-				console.log('using', String(usedMemPercent).trim(), 
+				console.log('using', String(usedMemPercent).trim(),
 					'percent memory, stopping runner creation!', args.worker.availrunners.length, 
 					'created on ', args.worker.name
 				);
@@ -260,7 +256,12 @@ var workers = (function(){
 					name: name,
 					worker: args.worker,
 					label: args.worker.name + ':' + name,
-
+					afterFreed: function(runner){
+						delete workers.runnerMap[runner.label];
+						console.log(`Runner freed ${runner.label}.`, runner.worker);
+						// Why does this need to run here?
+						workers.startRunners({worker: runner.worker});
+					}
 				});
 
 				args.onStart(runner, args);
@@ -299,16 +300,19 @@ var workers = (function(){
 		workers.checkForZombies();
 
 		// if there are workers being created, stop scale up and down check
-		if(workers.currentCreating+workers.length < workers.settings.min) null;
-		else if(workers.currentCreating)
+		if(workers.currentCreating+workers.length < workers.settings.min) {
+			null;
+		} else if(workers.currentCreating){
 			return console.log(`Killing balance, workers are being created.`);
+		}
 
 		// hold amount of workers with no used runners
 		var lastMinAval = 0;
 
 		// check to make sure the `workers.settings.minAvail` have free runners
 		for(let worker of workers.slice(-workers.settings.minAvail)){
-			if(worker.usedrunners === 0){
+			// INVERT this conditional
+			if(worker.usedrunners !== 0){
 				lastMinAval++;
 			}else{
 				// no need to keep counting, workers need to be created
@@ -353,7 +357,7 @@ var workers = (function(){
 			workers.push(worker);
 		});
 	};
-
+	// does this have to be last?
 	// make sure Digital Ocean has a tag for the current worker version
 	doapi.tagCreate(tagPrefix + workers.settings.version);
 
-- 
2.34.1


From e94308f6ac68fd48aea1505735033a01a052b314 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 01:48:55 -0400
Subject: [PATCH 05/42] updating workers.destroy

---
 routes/worker_manager.js | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/routes/worker_manager.js b/routes/worker_manager.js
index 01f6a47..8ba8d33 100644
--- a/routes/worker_manager.js
+++ b/routes/worker_manager.js
@@ -79,10 +79,17 @@ var Worker = (function(){
 			callback
 		);
 	};
+
+	proto.destroy = function(){
+		var worker = this;
+		return doapi.dropletDestroy(this.id, function(body) {
+			console.log('Deleted worker', this.name);
+		});
+	};
 })();
 
 
-var workers = (function(){
+var WorkerCollection = (function(){
 	// works array constructor. This will hold the works(order by creation) and all
 	// the methods interacting with the workers.
 	
@@ -181,13 +188,19 @@ var workers = (function(){
 	};
 
 	workers.destroy = function(worker){
+		// removes last one 
 		// todo: If worker is passed, check for it in the workers array and
 		// remove it if found.
-
-		var worker = worker || workers.pop();
-		return doapi.dropletDestroy(worker.id, function(body) {
-			console.log('Deleted worker', worker.name);
-		});
+		if ( worker ){
+			var worker_idx = workers.indexOf(worker);
+			if (~worker_idx){
+				workers.splice(worker_idx, 1);
+				return worker.destroy();
+			}
+		} else {
+			worker = workers.pop();
+			return worker.destroy();
+		}
 	};
 
 	workers.destroyByTag = function(tag){
@@ -366,4 +379,4 @@ var workers = (function(){
 })();
 
 
-module.exports = workers;
\ No newline at end of file
+module.exports = WorkerCollection;
\ No newline at end of file
-- 
2.34.1


From 87ee3a5fd15036095b0cf3366f968aa8226da457 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 01:51:28 -0400
Subject: [PATCH 06/42] updating file name

---
 routes/api.js                                      | 2 +-
 routes/{worker_manager.js => worker_collection.js} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename routes/{worker_manager.js => worker_collection.js} (100%)

diff --git a/routes/api.js b/routes/api.js
index 36546cd..8e2a51a 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -7,7 +7,7 @@ var request = require('request');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
 
-var workers = require('./worker_manager.js');
+var workers = require('./worker_collection.js');
 
 
 var attemptRun = function(req, res, runner, count){
diff --git a/routes/worker_manager.js b/routes/worker_collection.js
similarity index 100%
rename from routes/worker_manager.js
rename to routes/worker_collection.js
-- 
2.34.1


From 43caeb73f0567565a6c0bd98851dce0e07539b59 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 01:54:11 -0400
Subject: [PATCH 07/42] adding iffy

---
 routes/api.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index 8e2a51a..3b785f1 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -9,6 +9,11 @@ var doapi = require('../doapi')();
 
 var workers = require('./worker_collection.js');
 
+(function(){
+	console.log('========STARTING===========')
+	setInterval(workers.checkBalance, 15000);
+	workers.destroyByTag();
+})()
 
 var attemptRun = function(req, res, runner, count){
 	count = count || 0;
@@ -62,11 +67,6 @@ var attemptRun = function(req, res, runner, count){
 	});
 };
 
-console.log('========STARTING===========')
-// TODO: Make this a function
-setInterval(workers.checkBalance, 15000);
-workers.destroyByTag();
-
 // Why is this a GET?
 router.get('/stop/:name', function(req, res, next){
 	return lxc.stop(req.params.name, function(data){
-- 
2.34.1


From 6a11a0baa5cd2de5c82ce06296199cdd5a29422f Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 16:25:40 -0400
Subject: [PATCH 08/42] moved core of startRunners to Worker class.

---
 routes/worker_collection.js | 150 ++++++++++++++++++++----------------
 1 file changed, 83 insertions(+), 67 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 8ba8d33..73bd6ec 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -8,23 +8,28 @@ var settings = require('./workers.json');
 
 var Runner = (function(){
 	var proto = {};
-
+	var __empty = function(){};
+	
 	proto.create = function(config){
 		var runner = Object.create(proto);
-		var __empty = function(){};
 		Object.assign(runner, config);
-		runner.afterFreed = runner.afterFreed || __empty;
+		runner.cleanUp = __empty;
 		return runner;
 	};
 
-	proto.free = function(callback){
+	proto.free = function(){
 		var runner = this;
 		lxc.stop(runner.name, runner.worker.ip);
 		runner.worker.usedrunners--;
 		if(runner.hasOwnProperty('timeout')){
 			clearTimeout(runner.timeout);
-		};
-		runner.afterFreed();
+		}
+
+		if(runner.hasOwnProperty('cleanUp')){
+			runner.cleanUp();
+		}
+
+		runner.worker.startRunners();
 	};
 
 	proto.setTimeout = function(time){
@@ -44,6 +49,10 @@ var Runner = (function(){
 
 var Worker = (function(){
 	var proto = {};
+	var __empty = function(){};
+	
+	// settings should probably be retrieved via a function 
+	proto.settings = settings;
 
 	proto.create = function(config){
 		var worker = Object.create(proto);
@@ -86,6 +95,59 @@ var Worker = (function(){
 			console.log('Deleted worker', this.name);
 		});
 	};
+
+	proto.startRunners = function(args){
+		// console.log('starting runners on', args.worker.name, args.worker.ip)
+		
+		var worker = this;
+		// dont make runners on out dated workers
+		if(!worker || worker.settings.image > worker.image.id){
+			console.log(`Blocked outdated worker(${worker.image.id}), current image ${args.settings.image}.`)
+			return ;
+		}
+
+		args = args || {};
+		// percent of used RAM to stop runner creation
+		args.stopPercent = args.stopPercent || 80;
+		args.onStart = args.onStart || __empty;
+		args.onDone = args.onDone || __empty;
+
+
+		worker.ramPercentUsed(function(usedMemPercent){
+			if(usedMemPercent > args.stopPercent ){
+				console.log('using', String(usedMemPercent).trim(),
+					'percent memory, stopping runner creation!', worker.availrunners.length, 
+					'created on ', worker.name
+				);
+				args.onDone(args);
+				return ;
+			}
+
+			var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
+			// console.log('Free ram check passed!')
+			lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
+				if(!data.ip){
+					return setTimeout(worker.startRunners, 0, args);
+				} else {
+
+					// console.log('started runner on', args.worker.name)
+
+					var runner = Runner.create({
+						ip: data.ip,
+						name: name,
+						worker: worker,
+						label: worker.name + ':' + name
+					});
+
+					args.onStart(runner, args);
+
+					worker.availrunners.push(runner);
+
+					setTimeout(worker.startRunners, 0, args);
+				}
+			});
+		});
+	};
 })();
 
 
@@ -114,10 +176,17 @@ var WorkerCollection = (function(){
 	// about 3 minutes to create a worker.
 	workers.currentCreating = 0;
 
-
+	workers.__runnerCleanUp = function(label){
+		delete workers.runnerMap[label];
+	};
 	
 	workers.setRunner = function(runner){
 		runnerMap[runner.label] = runner;
+		var __empty = runner.cleanUp;
+		runner.cleanUp = function(){
+			workers.__runnerCleanUp(runner.label);
+			runner.cleanUp = __empty;
+		};
 	};
 
 
@@ -130,7 +199,6 @@ var WorkerCollection = (function(){
 		for(let worker of workers){
 			if(worker.availrunners.length === 0) continue;
 			if(runner && runner.worker.index <= worker.index) break;
-			// if(runner) runnerFree(runner);
 			if(runner) runner.free();
 
 			return worker.getRunner();
@@ -148,6 +216,8 @@ var WorkerCollection = (function(){
 
 		config = config || workers.settings;
 
+		// move this to Create Droplet function?
+
 		doapi.dropletToActive({
 			name: 'clw' + config.version + '-' + (Math.random()*100).toString().slice(-4),
 			image: config.image,
@@ -159,10 +229,10 @@ var WorkerCollection = (function(){
 				);
 			},
 			onActive: function(data, args){
-				workers.startRunners({
-					worker: Worker.create(data),
+				var worker = Worker.create(data);
+				worker.startRunners({
 					onStart: function(runner, args){
-						workers.push(args.worker);
+						workers.push(worker);
 						doapi.domianAddRecord({
 							domain: "codeland.us",
 							type: "A",
@@ -232,60 +302,6 @@ var WorkerCollection = (function(){
 		});
 	};
 
-	workers.startRunners = function(args){ 
-		// console.log('starting runners on', args.worker.name, args.worker.ip)
-
-		// dont make runners on out dated workers
-		if(!args.worker || workers.settings.image > args.worker.image.id){
-			console.log(`Blocked outdated worker(${args.worker.image.id}), current image ${workers.settings.image}.`)
-			return ;
-		}
-
-		// percent of used RAM to stop runner creation
-		args.stopPercent = args.stopPercent || 80;
-		args.onStart = args.onStart || function(){};
-		args.onDone = args.onDone || function(){};
-
-		args.worker.ramPercentUsed(function(usedMemPercent){
-			if(usedMemPercent > args.stopPercent ){
-				console.log('using', String(usedMemPercent).trim(),
-					'percent memory, stopping runner creation!', args.worker.availrunners.length, 
-					'created on ', args.worker.name
-				);
-				args.onDone(args);
-				return ;
-			}
-
-			var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
-			// console.log('Free ram check passed!')
-			lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
-				if(!data.ip){
-					return setTimeout(workers.startRunners, 0, args);
-				}
-				// console.log('started runner on', args.worker.name)
-
-				var runner = Runner.create({
-					ip: data.ip,
-					name: name,
-					worker: args.worker,
-					label: args.worker.name + ':' + name,
-					afterFreed: function(runner){
-						delete workers.runnerMap[runner.label];
-						console.log(`Runner freed ${runner.label}.`, runner.worker);
-						// Why does this need to run here?
-						workers.startRunners({worker: runner.worker});
-					}
-				});
-
-				args.onStart(runner, args);
-
-				args.worker.availrunners.push(runner);
-
-				setTimeout(workers.startRunners, 0, args);
-			});
-		});
-	};
-
 	workers.checkForZombies = function(){
 		// check to make sure all works are used or usable.
 		
@@ -313,7 +329,7 @@ var WorkerCollection = (function(){
 		workers.checkForZombies();
 
 		// if there are workers being created, stop scale up and down check
-		if(workers.currentCreating+workers.length < workers.settings.min) {
+		if(workers.currentCreating + workers.length < workers.settings.min) {
 			null;
 		} else if(workers.currentCreating){
 			return console.log(`Killing balance, workers are being created.`);
@@ -334,7 +350,7 @@ var WorkerCollection = (function(){
 		}
 
 		if(lastMinAval > workers.settings.minAvail){
-			// Remove workers if there are more then the settings states
+			// Remove workers if there are more than the settings states
 			console.log(
 				`Last ${workers.settings.minAvail} workers not used, killing last worker`, 
 				'lastMinAval:', lastMinAval,
-- 
2.34.1


From 983b76afce891cc857272dfa7ec64cedc73bfa33 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 16:44:48 -0400
Subject: [PATCH 09/42] Moving some code. - grouped runnerMap code in workers
 object - created isZombie function in Worker

---
 routes/worker_collection.js | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 73bd6ec..d082890 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -96,6 +96,10 @@ var Worker = (function(){
 		});
 	};
 
+	proto.isZombie = function(){
+		return this.availrunners.length === 0 && this.usedrunners === 0;
+	};
+
 	proto.startRunners = function(args){
 		// console.log('starting runners on', args.worker.name, args.worker.ip)
 		
@@ -160,7 +164,6 @@ var WorkerCollection = (function(){
 
 	var tagPrefix = settings.tagPrefix || 'clwV';
 
-	workers.runnerMap = {};
 
 	// persistent settings object
 	// .image is the currently used Digital Ocean snap shot ID
@@ -176,6 +179,11 @@ var WorkerCollection = (function(){
 	// about 3 minutes to create a worker.
 	workers.currentCreating = 0;
 
+	//**************************************************
+	//**************************************************
+	// temporary until a better location is found
+	workers.runnerMap = {};
+
 	workers.__runnerCleanUp = function(label){
 		delete workers.runnerMap[label];
 	};
@@ -194,6 +202,8 @@ var WorkerCollection = (function(){
 		return runnerMap[runner.label];
 	};
 
+	//**************************************************
+	//**************************************************
 
 	workers.getAvailableRunner = function(runner){
 		for(let worker of workers){
@@ -259,7 +269,7 @@ var WorkerCollection = (function(){
 
 	workers.destroy = function(worker){
 		// removes last one 
-		// todo: If worker is passed, check for it in the workers array and
+		// X TODO: If worker is passed, check for it in the workers array and
 		// remove it if found.
 		if ( worker ){
 			var worker_idx = workers.indexOf(worker);
@@ -290,6 +300,7 @@ var WorkerCollection = (function(){
 			});
 		}
 
+		// TODO: move to seperate method
 		doapi.dropletsByTag(tag, function(data){
 			data = JSON.parse(data);
 			console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
@@ -312,8 +323,8 @@ var WorkerCollection = (function(){
 			// if a runner has no available runners and no used runners, its a
 			// zombie. This should happen when a newer image ID has been added
 			// and old workers slowly lose there usefulness.
-			if(worker.availrunners.length === 0 && worker.usedrunners === 0){
-				workers.splice(workers.indexOf(worker), 1);
+
+			if(worker.isZombie()){
 				console.log(`Zombie! Worker ${worker.name}, destroying.`);
 				workers.destroy(worker);
 				zombies++;
-- 
2.34.1


From 482fe08c07865eb805ee40aa20341ad0fe7cb473 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 10 Oct 2017 23:31:02 -0400
Subject: [PATCH 10/42] patches for testing.

---
 routes/api.js               |  2 +-
 routes/worker_collection.js | 25 ++++++++++++++++++-------
 routes/workers.json         |  3 ++-
 testAPI.js                  |  2 +-
 4 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index 3b785f1..7e809b9 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -107,7 +107,7 @@ router.get('/ping/:runner', function(req, res, next){
 
 router.post('/run/:runner?', function (req, res, next){
 	console.log(`Request runner route!`);
-	var runner = workers.getAvailrunner(workers.getRunner(req.params.runner));
+	var runner = workers.getAvailableRunner(workers.getRunner(req.params.runner));
 	return attemptRun(req, res, runner);
 });
 
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index d082890..c57e9ea 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -44,6 +44,7 @@ var Runner = (function(){
 		}, time);
 	};
 
+	return proto;
 })();
 
 
@@ -56,7 +57,7 @@ var Worker = (function(){
 
 	proto.create = function(config){
 		var worker = Object.create(proto);
-
+		Object.assign(worker, config);
 		worker.networks.v4.forEach(function(value){
 			worker[value.type+'IP'] = value.ip_address;
 		});
@@ -64,7 +65,8 @@ var Worker = (function(){
 		worker.availrunners = [];
 		worker.ip = worker.publicIP;
 		worker.usedrunners = 0;
-		worker.index = workers.length;
+		worker.age = +(new Date());
+		console.log("AGE:",worker.age);
 
 		return worker;
 	};
@@ -131,7 +133,9 @@ var Worker = (function(){
 			// console.log('Free ram check passed!')
 			lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
 				if(!data.ip){
-					return setTimeout(worker.startRunners, 0, args);
+					return setTimeout(function(){
+						worker.startRunners(args);
+					}, 0);
 				} else {
 
 					// console.log('started runner on', args.worker.name)
@@ -147,11 +151,15 @@ var Worker = (function(){
 
 					worker.availrunners.push(runner);
 
-					setTimeout(worker.startRunners, 0, args);
+					setTimeout(function(){
+							worker.startRunners(args);
+					}, 0);
 				}
 			});
 		});
 	};
+
+	return proto;
 })();
 
 
@@ -189,7 +197,7 @@ var WorkerCollection = (function(){
 	};
 	
 	workers.setRunner = function(runner){
-		runnerMap[runner.label] = runner;
+		workers.runnerMap[runner.label] = runner;
 		var __empty = runner.cleanUp;
 		runner.cleanUp = function(){
 			workers.__runnerCleanUp(runner.label);
@@ -199,7 +207,7 @@ var WorkerCollection = (function(){
 
 
 	workers.getRunner = function(label){
-		return runnerMap[runner.label];
+		return workers.runnerMap[label];
 	};
 
 	//**************************************************
@@ -208,7 +216,7 @@ var WorkerCollection = (function(){
 	workers.getAvailableRunner = function(runner){
 		for(let worker of workers){
 			if(worker.availrunners.length === 0) continue;
-			if(runner && runner.worker.index <= worker.index) break;
+			if(runner && runner.worker.age <= worker.age) break;
 			if(runner) runner.free();
 
 			return worker.getRunner();
@@ -239,9 +247,12 @@ var WorkerCollection = (function(){
 				);
 			},
 			onActive: function(data, args){
+				// TODO: NO GOOD
+				data.index = workers.length;
 				var worker = Worker.create(data);
 				worker.startRunners({
 					onStart: function(runner, args){
+						// TODO: NO GOOD
 						workers.push(worker);
 						doapi.domianAddRecord({
 							domain: "codeland.us",
diff --git a/routes/workers.json b/routes/workers.json
index 0b060ae..ab85951 100644
--- a/routes/workers.json
+++ b/routes/workers.json
@@ -5,5 +5,6 @@
 	"size":"512mb",
 	"max":100,
 	"min":3,
-	"minAvail":3
+	"minAvail":3,
+	"tagPrefix": "clams-man-clams"
 }
\ No newline at end of file
diff --git a/testAPI.js b/testAPI.js
index 01b0ca2..a17e4c0 100644
--- a/testAPI.js
+++ b/testAPI.js
@@ -21,7 +21,7 @@ var callRunner = (function(){
 
 			
 			let httpOptions = {
-				url: 'http://codeland.bytedev.co:2000/api/run?once=true',
+				url: 'http://localhost:2000/api/run?once=true',
 				form: {
 					code: code || `python3 -c "
 from time import sleep
-- 
2.34.1


From 099bce38462d09f66f7cbcf97c18ad8e21dffca4 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 11 Oct 2017 00:29:50 -0400
Subject: [PATCH 11/42] patches for zombie check

---
 routes/worker_collection.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index c57e9ea..0f72581 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -66,7 +66,7 @@ var Worker = (function(){
 		worker.ip = worker.publicIP;
 		worker.usedrunners = 0;
 		worker.age = +(new Date());
-		console.log("AGE:",worker.age);
+		worker.isBuildingRunners = false;
 
 		return worker;
 	};
@@ -94,12 +94,12 @@ var Worker = (function(){
 	proto.destroy = function(){
 		var worker = this;
 		return doapi.dropletDestroy(this.id, function(body) {
-			console.log('Deleted worker', this.name);
+			console.log('Deleted worker', worker.name);
 		});
 	};
 
 	proto.isZombie = function(){
-		return this.availrunners.length === 0 && this.usedrunners === 0;
+		return this.availrunners.length === 0 && this.usedrunners === 0 && !this.isBuildingRunners;
 	};
 
 	proto.startRunners = function(args){
@@ -111,16 +111,16 @@ var Worker = (function(){
 			console.log(`Blocked outdated worker(${worker.image.id}), current image ${args.settings.image}.`)
 			return ;
 		}
-
 		args = args || {};
 		// percent of used RAM to stop runner creation
 		args.stopPercent = args.stopPercent || 80;
 		args.onStart = args.onStart || __empty;
 		args.onDone = args.onDone || __empty;
 
-
+		worker.isBuildingRunners = true;
 		worker.ramPercentUsed(function(usedMemPercent){
 			if(usedMemPercent > args.stopPercent ){
+				worker.isBuildingRunners = false;
 				console.log('using', String(usedMemPercent).trim(),
 					'percent memory, stopping runner creation!', worker.availrunners.length, 
 					'created on ', worker.name
-- 
2.34.1


From 654628b44d6414ccd2f7faf2149f67659096242b Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 11 Oct 2017 02:06:21 -0400
Subject: [PATCH 12/42] adding changes

---
 routes/worker_collection.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 0f72581..112e99a 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -371,6 +371,11 @@ var WorkerCollection = (function(){
 			}
 		}
 
+		console.log(
+			`LMA: ${lastMinAval}`,
+			`Settings MA: ${workers.settings.minAvail}`,
+			`Workers: ${workers.length}`
+		);
 		if(lastMinAval > workers.settings.minAvail){
 			// Remove workers if there are more than the settings states
 			console.log(
@@ -395,6 +400,7 @@ var WorkerCollection = (function(){
 		}
 
 	};
+	
 	workers.settingsSave = function(){
 		// save the live settings file to disk
 
-- 
2.34.1


From 41874764bda86b12440cac350e56ecb71ba75587 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 11 Oct 2017 22:59:23 -0400
Subject: [PATCH 13/42] updated checkBalance

---
 routes/worker_collection.js | 89 +++++++++++++++++++++----------------
 testAPI.js                  |  7 ++-
 2 files changed, 56 insertions(+), 40 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 112e99a..f2e592e 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -59,7 +59,7 @@ var Worker = (function(){
 		var worker = Object.create(proto);
 		Object.assign(worker, config);
 		worker.networks.v4.forEach(function(value){
-			worker[value.type+'IP'] = value.ip_address;
+			worker[ value.type + 'IP' ] = value.ip_address;
 		});
 
 		worker.availrunners = [];
@@ -67,6 +67,7 @@ var Worker = (function(){
 		worker.usedrunners = 0;
 		worker.age = +(new Date());
 		worker.isBuildingRunners = false;
+		worker.canSchedule = true;
 
 		return worker;
 	};
@@ -93,6 +94,7 @@ var Worker = (function(){
 
 	proto.destroy = function(){
 		var worker = this;
+		worker.canSchedule = false;
 		return doapi.dropletDestroy(this.id, function(body) {
 			console.log('Deleted worker', worker.name);
 		});
@@ -103,7 +105,7 @@ var Worker = (function(){
 	};
 
 	proto.startRunners = function(args){
-		// console.log('starting runners on', args.worker.name, args.worker.ip)
+		console.log('Starting runners on', args.worker.name, args.worker.ip);
 		
 		var worker = this;
 		// dont make runners on out dated workers
@@ -189,7 +191,7 @@ var WorkerCollection = (function(){
 
 	//**************************************************
 	//**************************************************
-	// temporary until a better location is found
+	// TODO: Move to Runners
 	workers.runnerMap = {};
 
 	workers.__runnerCleanUp = function(label){
@@ -237,7 +239,7 @@ var WorkerCollection = (function(){
 		// move this to Create Droplet function?
 
 		doapi.dropletToActive({
-			name: 'clw' + config.version + '-' + (Math.random()*100).toString().slice(-4),
+			name: config.tagPrefix + config.version + '-' + (Math.random()*100).toString().slice(-4),
 			image: config.image,
 			size: config.size,
 			onCreate: function(data){
@@ -351,56 +353,65 @@ var WorkerCollection = (function(){
 		workers.checkForZombies();
 
 		// if there are workers being created, stop scale up and down check
-		if(workers.currentCreating + workers.length < workers.settings.min) {
+		if(workers.currentCreating + workers.length > workers.settings.min) {
 			null;
 		} else if(workers.currentCreating){
 			return console.log(`Killing balance, workers are being created.`);
 		}
 
-		// hold amount of workers with no used runners
-		var lastMinAval = 0;
+		// count workers and locate oldest worker
+		var oldestWorker, isNotOlder, workerCount = 0;
 
-		// check to make sure the `workers.settings.minAvail` have free runners
-		for(let worker of workers.slice(-workers.settings.minAvail)){
-			// INVERT this conditional
-			if(worker.usedrunners !== 0){
-				lastMinAval++;
-			}else{
-				// no need to keep counting, workers need to be created
-				break;
+		for(let worker of workers){
+			console.log(`
+				CHECK_BALANCE
+				worker.name: ${worker.name}
+				worker.usedrunners: ${worker.usedrunners}
+				worker.availrunners: ${worker.availrunners.length}
+				workerCount: ${workerCount}
+				compare: ${worker.usedrunners !== 0}
+			`);
+
+			if(worker.usedrunners === 0){
+				workerCount++;
+				isNotOlder = oldestWorker && oldestWorker.age < worker.age
+				oldestWorker = (isNotOlder ? oldestWorker:worker);
 			}
 		}
 
-		console.log(
-			`LMA: ${lastMinAval}`,
-			`Settings MA: ${workers.settings.minAvail}`,
-			`Workers: ${workers.length}`
-		);
-		if(lastMinAval > workers.settings.minAvail){
-			// Remove workers if there are more than the settings states
-			console.log(
-				`Last ${workers.settings.minAvail} workers not used, killing last worker`, 
-				'lastMinAval:', lastMinAval,
-				'minAvail:', workers.settings.minAvail,
-				'workers:', workers.length
-			);
+		if(workerCount > workers.settings.minAvail){
+			// Remove oldest worker if there are more than the settings file state
+			console.log(`
+				Destroying Worker
+				Last ${workers.settings.minAvail} workers not used, killing last worker
+				workerCount: ${workerCount}
+				minAvail: ${workers.settings.minAvail}
+				workers: ${workers.length}
+			`);
+			return workers.destroy(oldestWorker);
 
-			return workers.destroy();
-
-		} else if(lastMinAval < workers.settings.minAvail){
-			// creates workers if the settings file demands it
-			console.log(
-				'last 3 workers have no free runners, starting worker',
-				'lastMinAval:', lastMinAval,
-				'minAvail:', workers.settings.minAvail,
-				'workers:', workers.length
-			);
+		} else if( workerCount < workers.settings.minAvail){
+			// Creates worker if there are less than the settings state
+			console.log(`
+				Creating Worker
+				last 3 workers have no free runners, starting worker,
+				workerCount: ${workerCount}
+				minAvail: ${workers.settings.minAvail}
+				workers: ${workers.length}
+			`);
 
 			return workers.create();
+		} else {
+			console.log(`
+				Blanced
+				LMA: ${workerCount}
+				Settings MA: ${workers.settings.minAvail}
+				Workers: ${workers.length}
+			`);
 		}
 
 	};
-	
+
 	workers.settingsSave = function(){
 		// save the live settings file to disk
 
diff --git a/testAPI.js b/testAPI.js
index a17e4c0..91ff8f8 100644
--- a/testAPI.js
+++ b/testAPI.js
@@ -37,6 +37,11 @@ sleep(${sleepTime})
 					noRunner++;
 				}else if(error || response.statusCode !== 200){
 					errors++;
+					console.log(`
+						ID: ${id} 
+						Error: ${error}
+					`);
+				} else {	
 					body = JSON.parse(body);
 					res = (Buffer.from(body.res, 'base64').toString('ascii'));
 				}
@@ -58,4 +63,4 @@ let __do = function(till){
 	setTimeout(__do, 1500, --till);
 };
 
-__do(30)
+__do(500);
-- 
2.34.1


From 11e44c94e76c0ccb0e3b5f14cd9932efa1c53862 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 11 Oct 2017 23:10:19 -0400
Subject: [PATCH 14/42] Removed odd use of null

---
 routes/worker_collection.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index f2e592e..f2cceec 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -353,9 +353,7 @@ var WorkerCollection = (function(){
 		workers.checkForZombies();
 
 		// if there are workers being created, stop scale up and down check
-		if(workers.currentCreating + workers.length > workers.settings.min) {
-			null;
-		} else if(workers.currentCreating){
+		if(workers.currentCreating + workers.length >= workers.settings.min){
 			return console.log(`Killing balance, workers are being created.`);
 		}
 
-- 
2.34.1


From a358a5212a05b61d962ad50ee83d014882c4afd5 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 11 Oct 2017 23:25:17 -0400
Subject: [PATCH 15/42] Simplifing balance

---
 routes/worker_collection.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index f2cceec..7d424aa 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -353,7 +353,8 @@ var WorkerCollection = (function(){
 		workers.checkForZombies();
 
 		// if there are workers being created, stop scale up and down check
-		if(workers.currentCreating + workers.length >= workers.settings.min){
+		var skipBalance = workers.currentCreating + workers.length >= workers.settings.min
+		if(skipBalance || workers.currentCreating){
 			return console.log(`Killing balance, workers are being created.`);
 		}
 
-- 
2.34.1


From a1a85b2c69de9e375d9b5b361aba1e49bf51ab6e Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Thu, 12 Oct 2017 00:58:13 -0400
Subject: [PATCH 16/42] patch

---
 routes/worker_collection.js | 6 +++---
 testAPI.js                  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 7d424aa..e0b4b5b 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -105,9 +105,9 @@ var Worker = (function(){
 	};
 
 	proto.startRunners = function(args){
-		console.log('Starting runners on', args.worker.name, args.worker.ip);
-		
 		var worker = this;
+		
+		console.log('Starting runners on', worker.name, worker.ip);
 		// dont make runners on out dated workers
 		if(!worker || worker.settings.image > worker.image.id){
 			console.log(`Blocked outdated worker(${worker.image.id}), current image ${args.settings.image}.`)
@@ -354,7 +354,7 @@ var WorkerCollection = (function(){
 
 		// if there are workers being created, stop scale up and down check
 		var skipBalance = workers.currentCreating + workers.length >= workers.settings.min
-		if(skipBalance || workers.currentCreating){
+		if(workers.currentCreating && skipBalance){
 			return console.log(`Killing balance, workers are being created.`);
 		}
 
diff --git a/testAPI.js b/testAPI.js
index 91ff8f8..30b62cd 100644
--- a/testAPI.js
+++ b/testAPI.js
@@ -63,4 +63,4 @@ let __do = function(till){
 	setTimeout(__do, 1500, --till);
 };
 
-__do(500);
+__do(1000);
-- 
2.34.1


From 4afb69cdf7d3004016b504262c9303a693f3fba9 Mon Sep 17 00:00:00 2001
From: William Mantly <wmantly@gmail.com>
Date: Thu, 12 Oct 2017 01:15:33 -0400
Subject: [PATCH 17/42] compounding creating runners bug

---
 routes/worker_collection.js | 4 ++--
 routes/workers.json         | 2 +-
 testAPI.js                  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index e0b4b5b..0aa9e0d 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -109,7 +109,7 @@ var Worker = (function(){
 		
 		console.log('Starting runners on', worker.name, worker.ip);
 		// dont make runners on out dated workers
-		if(!worker || worker.settings.image > worker.image.id){
+		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
 			console.log(`Blocked outdated worker(${worker.image.id}), current image ${args.settings.image}.`)
 			return ;
 		}
@@ -363,7 +363,7 @@ var WorkerCollection = (function(){
 
 		for(let worker of workers){
 			console.log(`
-				CHECK_BALANCE
+				Checking worker
 				worker.name: ${worker.name}
 				worker.usedrunners: ${worker.usedrunners}
 				worker.availrunners: ${worker.availrunners.length}
diff --git a/routes/workers.json b/routes/workers.json
index ab85951..8862016 100644
--- a/routes/workers.json
+++ b/routes/workers.json
@@ -6,5 +6,5 @@
 	"max":100,
 	"min":3,
 	"minAvail":3,
-	"tagPrefix": "clams-man-clams"
+	"tagPrefix": "devCLW"
 }
\ No newline at end of file
diff --git a/testAPI.js b/testAPI.js
index 30b62cd..0910155 100644
--- a/testAPI.js
+++ b/testAPI.js
@@ -63,4 +63,4 @@ let __do = function(till){
 	setTimeout(__do, 1500, --till);
 };
 
-__do(1000);
+__do(4);
-- 
2.34.1


From c5e75967d9356e3024516cf91b347b2e856971cc Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Thu, 12 Oct 2017 01:26:32 -0400
Subject: [PATCH 18/42] patching log bug

---
 routes/worker_collection.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 0aa9e0d..e82e309 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -110,7 +110,7 @@ var Worker = (function(){
 		console.log('Starting runners on', worker.name, worker.ip);
 		// dont make runners on out dated workers
 		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
-			console.log(`Blocked outdated worker(${worker.image.id}), current image ${args.settings.image}.`)
+			console.log(`Blocked outdated worker(${worker.image.id}), current image ${worker.settings.image}.`)
 			return ;
 		}
 		args = args || {};
-- 
2.34.1


From 4b6aacb76919c84ad5f109fcdf1a8858e8bb4e47 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Thu, 12 Oct 2017 21:50:49 -0400
Subject: [PATCH 19/42] moving runnerMap to Runner class

---
 routes/worker_collection.js | 36 +++++++++++++++++++++---------------
 testAPI.js                  |  7 ++++++-
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index e82e309..3542745 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -9,7 +9,23 @@ var settings = require('./workers.json');
 var Runner = (function(){
 	var proto = {};
 	var __empty = function(){};
+
+	proto.runnerMap = {};
+
+	proto.cleanUp = function(label){
+		delete proto.runnerMap[label];
+	};
 	
+	proto.set = function(runner){
+		proto.runnerMap[runner.label] = runner;
+		proto.runnerMap[runner.label] = runner;
+	};
+
+	proto.get = function(label){
+		return proto.runnerMap[label];
+	};
+	
+
 	proto.create = function(config){
 		var runner = Object.create(proto);
 		Object.assign(runner, config);
@@ -189,27 +205,16 @@ var WorkerCollection = (function(){
 	// about 3 minutes to create a worker.
 	workers.currentCreating = 0;
 
-	//**************************************************
-	//**************************************************
-	// TODO: Move to Runners
-	workers.runnerMap = {};
-
-	workers.__runnerCleanUp = function(label){
-		delete workers.runnerMap[label];
-	};
+	// REMOVE THIS 
+	worker.runnerMap = Runner.runnerMap;
 	
 	workers.setRunner = function(runner){
-		workers.runnerMap[runner.label] = runner;
-		var __empty = runner.cleanUp;
-		runner.cleanUp = function(){
-			workers.__runnerCleanUp(runner.label);
-			runner.cleanUp = __empty;
-		};
+		Runner.set(runner);
 	};
 
 
 	workers.getRunner = function(label){
-		return workers.runnerMap[label];
+		return Runner.get(label);
 	};
 
 	//**************************************************
@@ -316,6 +321,7 @@ var WorkerCollection = (function(){
 		// TODO: move to seperate method
 		doapi.dropletsByTag(tag, function(data){
 			data = JSON.parse(data);
+			console.log(data);
 			console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
 				data['droplets'].map(function(item){
 					return item.name+' | '+item.id;
diff --git a/testAPI.js b/testAPI.js
index 0910155..9366630 100644
--- a/testAPI.js
+++ b/testAPI.js
@@ -45,7 +45,12 @@ sleep(${sleepTime})
 					body = JSON.parse(body);
 					res = (Buffer.from(body.res, 'base64').toString('ascii'));
 				}
-				console.log(`${id} with results ${res}. Errors ${errors}. No runner ${noRunner}. Completed ${completed}`);
+				console.log(`
+					${id} with results ${res}. 
+					Errors ${errors}. 
+					No runner ${noRunner}. 
+					Completed ${completed}
+				`);
 
 				callback()
 
-- 
2.34.1


From a79e99433127d0e730bc39d5294d5070d30aee35 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Fri, 13 Oct 2017 00:06:46 -0400
Subject: [PATCH 20/42] debuging Worker.startRunners

---
 routes/worker_collection.js | 142 ++++++++++++++++++++----------------
 routes/workers.json         |   2 +-
 2 files changed, 82 insertions(+), 62 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 3542745..7894f54 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -4,6 +4,19 @@ var jsonfile = require('jsonfile');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
 var settings = require('./workers.json');
+settings.tagPrefix = settings.tagPrefix || 'clwV';
+
+var utils = (function(){
+	return {
+
+		"uuid": function(){
+			return (Math.random()*100).toString().slice(-4);
+		}
+
+	};
+
+})();
+
 
 
 var Runner = (function(){
@@ -120,13 +133,45 @@ var Worker = (function(){
 		return this.availrunners.length === 0 && this.usedrunners === 0 && !this.isBuildingRunners;
 	};
 
+	proto.register = function(){
+		var worker = this;
+		doapi.domianAddRecord({
+			domain: "codeland.us",
+			type: "A",
+			name: "*." + this.name + ".workers",
+			data: this.publicIP
+		});
+	};
+
+	proto.initialize = function(hooks, config){
+		// Create droplet
+		// Once active the droplet begins to create runners
+		doapi.dropletToActive({
+			name: config.tagPrefix + (config.version+"") + '-' + utils.uuid(),
+			image: config.image,
+			size: config.size,
+			onCreate: function(data){
+				doapi.dropletSetTag(
+					config.tagPrefix + config.version, 
+					data.droplet.id
+				);
+			},
+			onActive: function(data, args){
+				var worker = Worker.create(data);
+				worker.startRunners(hooks);
+			}
+		});
+	};
+
 	proto.startRunners = function(args){
 		var worker = this;
+
+		args.count = args.count || 0;
 		
-		console.log('Starting runners on', worker.name, worker.ip);
+		console.log('Starting runners on', worker.name, worker.ip, args.count);
 		// dont make runners on out dated workers
 		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
-			console.log(`Blocked outdated worker(${worker.image.id}), current image ${worker.settings.image}.`)
+			if(worker) console.log(`Blocked worker(${worker.image.id}), current image ${worker.settings.image}. Building: ${worker.isBuildingRunners}`)
 			return ;
 		}
 		args = args || {};
@@ -139,40 +184,41 @@ var Worker = (function(){
 		worker.ramPercentUsed(function(usedMemPercent){
 			if(usedMemPercent > args.stopPercent ){
 				worker.isBuildingRunners = false;
-				console.log('using', String(usedMemPercent).trim(),
+				console.log('---using', String(usedMemPercent).trim(),
 					'percent memory, stopping runner creation!', worker.availrunners.length, 
 					'created on ', worker.name
 				);
 				args.onDone(args);
 				return ;
 			}
+			console.log('+++using', String(usedMemPercent).trim(),
+				'percent memory on ', worker.name, 
+				'Runners:', worker.availrunners.length,
+				`Used? ${usedMemPercent}`
 
-			var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
+			);
+			var name = 'crunner-' + utils.uuid();
 			// console.log('Free ram check passed!')
 			lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
-				if(!data.ip){
-					return setTimeout(function(){
-						worker.startRunners(args);
-					}, 0);
-				} else {
-
-					// console.log('started runner on', args.worker.name)
+				if(data.ip){
+					console.log('started runner on', worker.name)
 
 					var runner = Runner.create({
-						ip: data.ip,
-						name: name,
-						worker: worker,
-						label: worker.name + ':' + name
+						"ip": data.ip,
+						"name": name,
+						"worker": worker,
+						"label": worker.name + ':' + name
 					});
 
-					args.onStart(runner, args);
+					args.onStart(worker, args);
 
 					worker.availrunners.push(runner);
-
-					setTimeout(function(){
-							worker.startRunners(args);
-					}, 0);
 				}
+
+				return setTimeout(function(){
+					worker.isBuildingRunners = false;
+					worker.startRunners(args);
+				}, 0);
 			});
 		});
 	};
@@ -188,8 +234,6 @@ var WorkerCollection = (function(){
 	// base array that will be the workers objects.
 	var workers = [];
 
-	var tagPrefix = settings.tagPrefix || 'clwV';
-
 
 	// persistent settings object
 	// .image is the currently used Digital Ocean snap shot ID
@@ -206,7 +250,7 @@ var WorkerCollection = (function(){
 	workers.currentCreating = 0;
 
 	// REMOVE THIS 
-	worker.runnerMap = Runner.runnerMap;
+	workers.runnerMap = Runner.runnerMap;
 	
 	workers.setRunner = function(runner){
 		Runner.set(runner);
@@ -236,46 +280,23 @@ var WorkerCollection = (function(){
 		// manages the creation of a work from first call to all runners seeded
 
 		// dont create more workers then the settings file allows
-		if(workers.currentCreating > workers.settings.max ) return false;
+		if(workers.length + workers.currentCreating >= workers.settings.max ) return false;
 		workers.currentCreating++;
 
 		config = config || workers.settings;
 
-		// move this to Create Droplet function?
-
-		doapi.dropletToActive({
-			name: config.tagPrefix + config.version + '-' + (Math.random()*100).toString().slice(-4),
-			image: config.image,
-			size: config.size,
-			onCreate: function(data){
-				doapi.dropletSetTag(
-					tagPrefix + config.version, 
-					data.droplet.id
-				);
+		var args = {
+			onStart: function(worker, args){
+				workers.push(worker);
+				worker.register();
+				args.onStart = function(){};
 			},
-			onActive: function(data, args){
-				// TODO: NO GOOD
-				data.index = workers.length;
-				var worker = Worker.create(data);
-				worker.startRunners({
-					onStart: function(runner, args){
-						// TODO: NO GOOD
-						workers.push(worker);
-						doapi.domianAddRecord({
-							domain: "codeland.us",
-							type: "A",
-							name: "*." + worker.name + ".workers",
-							data: worker.publicIP
-						});
-						args.onStart = function(){};
-					},
-					onDone: function(args){
-						console.log("Seeded runners on", worker.name);
-						workers.currentCreating--;
-					}
-				});
+			onDone: function(args){
+				console.log("Seeded runners on", worker.name);
+				workers.currentCreating--;
 			}
-		});
+		};
+		Worker.initialize(args, config);
 	};
 
 	workers.__workersId = function(argument){
@@ -304,7 +325,7 @@ var WorkerCollection = (function(){
 	workers.destroyByTag = function(tag){
 		// Delete works that with
 
-		tag = tag || tagPrefix + workers.settings.version;
+		tag = tag || workers.settings.tagPrefix + workers.settings.version;
 		let currentIDs = workers.__workersId();
 
 		let deleteDroplets = function(droplets){
@@ -321,7 +342,6 @@ var WorkerCollection = (function(){
 		// TODO: move to seperate method
 		doapi.dropletsByTag(tag, function(data){
 			data = JSON.parse(data);
-			console.log(data);
 			console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
 				data['droplets'].map(function(item){
 					return item.name+' | '+item.id;
@@ -354,7 +374,7 @@ var WorkerCollection = (function(){
 	};
 
 	workers.checkBalance = function(){
-		console.log(`Checking balance.`);
+		console.log(`${(new Date())} Checking balance.`);
 
 		workers.checkForZombies();
 
@@ -432,7 +452,7 @@ var WorkerCollection = (function(){
 	};
 	// does this have to be last?
 	// make sure Digital Ocean has a tag for the current worker version
-	doapi.tagCreate(tagPrefix + workers.settings.version);
+	doapi.tagCreate(workers.settings.tagPrefix + workers.settings.version);
 
 	return workers;
 
diff --git a/routes/workers.json b/routes/workers.json
index 8862016..3b22b85 100644
--- a/routes/workers.json
+++ b/routes/workers.json
@@ -6,5 +6,5 @@
 	"max":100,
 	"min":3,
 	"minAvail":3,
-	"tagPrefix": "devCLW"
+	"tagPrefix": "dev--CLW"
 }
\ No newline at end of file
-- 
2.34.1


From 0ac93e70f6746b12125246c085e4d25f458a3b5d Mon Sep 17 00:00:00 2001
From: William Mantly Jr <wmantly@gmail.com>
Date: Fri, 13 Oct 2017 00:43:48 -0400
Subject: [PATCH 21/42] Update lxc.js

---
 lxc.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxc.js b/lxc.js
index ab544d1..6018fe4 100644
--- a/lxc.js
+++ b/lxc.js
@@ -5,7 +5,7 @@ var exec = require('child_process').exec;
 function sysExec(command, ip, callback){
 	ip = ip || '104.236.77.157';
 	command = new Buffer(command).toString('base64')
-	command = 'ssh -i ~/.ssh/clw_rsa -o StrictHostKeyChecking=no virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"';
+	command = 'ssh -i ~/.ssh/clw_rsa -o "StrictHostKeyChecking no" virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"';
 	// command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command;
 
 	return exec(command, (function(callback){
-- 
2.34.1


From cac789fed38a9be55c52a195ee87a9f96230716e Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Fri, 13 Oct 2017 00:46:44 -0400
Subject: [PATCH 22/42] adding check before instantiating the runner

---
 routes/worker_collection.js | 57 ++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 26 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 7894f54..dcb4d18 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -165,10 +165,8 @@ var Worker = (function(){
 
 	proto.startRunners = function(args){
 		var worker = this;
-
-		args.count = args.count || 0;
 		
-		console.log('Starting runners on', worker.name, worker.ip, args.count);
+		console.log('Starting runners on', worker.name, worker.ip);
 		// dont make runners on out dated workers
 		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
 			if(worker) console.log(`Blocked worker(${worker.image.id}), current image ${worker.settings.image}. Building: ${worker.isBuildingRunners}`)
@@ -182,44 +180,51 @@ var Worker = (function(){
 
 		worker.isBuildingRunners = true;
 		worker.ramPercentUsed(function(usedMemPercent){
+			console.log(arguments);
 			if(usedMemPercent > args.stopPercent ){
 				worker.isBuildingRunners = false;
 				console.log('---using', String(usedMemPercent).trim(),
 					'percent memory, stopping runner creation!', worker.availrunners.length, 
 					'created on ', worker.name
 				);
-				args.onDone(args);
+				args.onDone(worker, args);
 				return ;
-			}
-			console.log('+++using', String(usedMemPercent).trim(),
-				'percent memory on ', worker.name, 
-				'Runners:', worker.availrunners.length,
-				`Used? ${usedMemPercent}`
+			} else if (usedMemPercent !== ""){
+				console.log('+++using', String(usedMemPercent).trim(),
+					'percent memory on ', worker.name, 
+					'Runners:', worker.availrunners.length,
+					`Used? ${usedMemPercent}`
 
-			);
-			var name = 'crunner-' + utils.uuid();
-			// console.log('Free ram check passed!')
-			lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
-				if(data.ip){
-					console.log('started runner on', worker.name)
+				);
+				var name = 'crunner-' + utils.uuid();
+				// console.log('Free ram check passed!')
+				lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
+					if(data.ip){
+						console.log('started runner on', worker.name);
 
-					var runner = Runner.create({
-						"ip": data.ip,
-						"name": name,
-						"worker": worker,
-						"label": worker.name + ':' + name
-					});
+						var runner = Runner.create({
+							"ip": data.ip,
+							"name": name,
+							"worker": worker,
+							"label": worker.name + ':' + name
+						});
 
-					args.onStart(worker, args);
+						args.onStart(worker, args);
 
-					worker.availrunners.push(runner);
-				}
+						worker.availrunners.push(runner);
+					}
 
+					return setTimeout(function(){
+						worker.isBuildingRunners = false;
+						worker.startRunners(args);
+					}, 0);
+				});
+			} else {
 				return setTimeout(function(){
 					worker.isBuildingRunners = false;
 					worker.startRunners(args);
 				}, 0);
-			});
+			}
 		});
 	};
 
@@ -291,7 +296,7 @@ var WorkerCollection = (function(){
 				worker.register();
 				args.onStart = function(){};
 			},
-			onDone: function(args){
+			onDone: function(worker, args){
 				console.log("Seeded runners on", worker.name);
 				workers.currentCreating--;
 			}
-- 
2.34.1


From ce31c7f1d5ac6f1f62557fc0caf79ca25d161eef Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 16 Oct 2017 12:01:44 -0400
Subject: [PATCH 23/42] updating startRunners command.

---
 allocate_runners.sh         | 27 +++++++++++++
 docs/lxc-world.md           | 19 ++++++++++
 docs/memory-managment.md    | 27 +++++++++++++
 routes/worker_collection.js | 76 +++++++++++++++++++++++++++----------
 testAPI.js                  |  2 +-
 5 files changed, 129 insertions(+), 22 deletions(-)
 create mode 100644 allocate_runners.sh
 create mode 100644 docs/lxc-world.md
 create mode 100644 docs/memory-managment.md

diff --git a/allocate_runners.sh b/allocate_runners.sh
new file mode 100644
index 0000000..86a1164
--- /dev/null
+++ b/allocate_runners.sh
@@ -0,0 +1,27 @@
+maxMemoryUsage=80;
+baseName="crunner0";
+namePrefix="cubs";
+runners="";
+
+function usedMemoryPercent () {
+	memoryAvailable=$(head /proc/meminfo|grep MemAvail|grep -Po '\d+');
+	totalMemory=$(head /proc/meminfo|grep MemTotal|grep -Po '\d+');
+	difference=$(expr $totalMemory - $memoryAvailable);
+	difference=$(expr $difference \* 100);
+	memory=$(expr $difference / $totalMemory);
+}
+
+
+usedMemoryPercent;
+until [[ $memory -gt $maxMemoryUsage ]]; do
+	
+	runnerName="${namePrefix}${RANDOM}";
+	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d
+	if[[ $? -eq 0 ]] 
+	then
+		runners="${runners};${runnerName}";
+	fi
+	usedMemoryPercent;
+done
+
+echo $runners
\ No newline at end of file
diff --git a/docs/lxc-world.md b/docs/lxc-world.md
new file mode 100644
index 0000000..3edc391
--- /dev/null
+++ b/docs/lxc-world.md
@@ -0,0 +1,19 @@
+##### LXC Commands
+
+
+`lxc-top`:
+
+	Realtime Meta data about containers(not machine friendly). Monitor container statistics.
+
+`lxc-ls --fancy`:
+
+	list of existing containers
+
+`lxc-start-ephemeral`, `lxc-copy`:
+
+	start an ephemeral copy of an existing container
+
+
+`lxc-attach`:
+
+	start a process inside a running container.(enter the droplet)
\ No newline at end of file
diff --git a/docs/memory-managment.md b/docs/memory-managment.md
new file mode 100644
index 0000000..79caf7d
--- /dev/null
+++ b/docs/memory-managment.md
@@ -0,0 +1,27 @@
+##### Memory Managment
+
+
+`free`, `free -h`: 
+
+	display amount of free and used memory in the system.
+	* Disclamers:
+		- DO NOT USE IN SCRIPT
+
+`/proc/`:
+	
+	It is actually an kernel application. Looks like a file, but is actually generated on demand by the kernel. All "files" in proc are kernal commands. Interact with these commands as if they were files. 
+	
+
+`/proc/meminfo`:
+	
+	File containing realtime memory info.
+
+`/proc/cpuinfo`:
+
+	cpu info
+
+
+
+`/dev`:
+	
+	DANGER This folder contains physical interfaces to hardware on or connected to the machine. Just don't. Read first. Except `/dev/null` it is a blackhole. 
\ No newline at end of file
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index dcb4d18..e7b96b3 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -4,6 +4,7 @@ var jsonfile = require('jsonfile');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
 var settings = require('./workers.json');
+var fs = require('fs');
 settings.tagPrefix = settings.tagPrefix || 'clwV';
 
 var utils = (function(){
@@ -42,7 +43,6 @@ var Runner = (function(){
 	proto.create = function(config){
 		var runner = Object.create(proto);
 		Object.assign(runner, config);
-		runner.cleanUp = __empty;
 		return runner;
 	};
 
@@ -58,7 +58,7 @@ var Runner = (function(){
 			runner.cleanUp();
 		}
 
-		runner.worker.startRunners();
+		runner.worker.newStartRunners();
 	};
 
 	proto.setTimeout = function(time){
@@ -143,7 +143,7 @@ var Worker = (function(){
 		});
 	};
 
-	proto.initialize = function(hooks, config){
+	proto.initialize = function(callback, config){
 		// Create droplet
 		// Once active the droplet begins to create runners
 		doapi.dropletToActive({
@@ -158,10 +158,48 @@ var Worker = (function(){
 			},
 			onActive: function(data, args){
 				var worker = Worker.create(data);
-				worker.startRunners(hooks);
+				worker.newStartRunners(callback);
 			}
 		});
 	};
+	proto.newStartRunners = function(args){
+		// onStart is not necessary
+		var worker = this;
+		args.stopPercent = args.stopPercent || 80;
+		args.callback = args.callback || __empty;
+
+		// dont make runners on out dated workers
+		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
+			if(worker) {
+				console.log(`
+					Blocked worker(${worker.image.id}), current image ${worker.settings.image}.
+					Building: ${worker.isBuildingRunners}
+				`);
+			}
+			return;
+		}
+
+		worker.isBuildingRunners = true;
+		fs.read(__dirname + "../allocate_runners.sh", function(error, data){
+			console.log(data);
+			// lxc.exec(data, function(output){
+			// 	// output chould be list of runner names
+			// 	console.log(output);
+			// 	// for name in output:
+			// 	// var runner = Runner.create({
+			// 	// 	"name": name,
+			// 	// 	"worker": worker,
+			// 	// 	"label": worker.name + ':' + name
+			// 	// });
+
+			// 	// worker.availrunners.push(runner);
+			// 	// end for
+			// 	worker.isBuildingRunners = false;
+			// 	args.callback(worker);
+			// });
+		});
+	};
+
 
 	proto.startRunners = function(args){
 		var worker = this;
@@ -179,6 +217,8 @@ var Worker = (function(){
 		args.onDone = args.onDone || __empty;
 
 		worker.isBuildingRunners = true;
+		
+		// no longer needed
 		worker.ramPercentUsed(function(usedMemPercent){
 			console.log(arguments);
 			if(usedMemPercent > args.stopPercent ){
@@ -199,6 +239,11 @@ var Worker = (function(){
 				var name = 'crunner-' + utils.uuid();
 				// console.log('Free ram check passed!')
 				lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
+					setTimeout(function(){
+						worker.isBuildingRunners = false;
+						worker.startRunners(args);
+					}, 0);
+
 					if(data.ip){
 						console.log('started runner on', worker.name);
 
@@ -214,10 +259,6 @@ var Worker = (function(){
 						worker.availrunners.push(runner);
 					}
 
-					return setTimeout(function(){
-						worker.isBuildingRunners = false;
-						worker.startRunners(args);
-					}, 0);
 				});
 			} else {
 				return setTimeout(function(){
@@ -289,19 +330,12 @@ var WorkerCollection = (function(){
 		workers.currentCreating++;
 
 		config = config || workers.settings;
-
-		var args = {
-			onStart: function(worker, args){
-				workers.push(worker);
-				worker.register();
-				args.onStart = function(){};
-			},
-			onDone: function(worker, args){
-				console.log("Seeded runners on", worker.name);
-				workers.currentCreating--;
-			}
-		};
-		Worker.initialize(args, config);
+		Worker.initialize(function(worker){
+			console.log("Seeded runners on", worker.name);
+			workers.push(worker);
+			worker.register();
+			workers.currentCreating--;
+		}, config);
 	};
 
 	workers.__workersId = function(argument){
diff --git a/testAPI.js b/testAPI.js
index 9366630..30b3c8c 100644
--- a/testAPI.js
+++ b/testAPI.js
@@ -68,4 +68,4 @@ let __do = function(till){
 	setTimeout(__do, 1500, --till);
 };
 
-__do(4);
+__do(1000);
-- 
2.34.1


From 337423d2948c6032cb8fa4d65f25897f5e7e9297 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 16 Oct 2017 17:33:10 -0400
Subject: [PATCH 24/42] adding newStartRunners command

---
 allocate_runners.sh         |  7 ++---
 routes/worker_collection.js | 61 ++++++++++++++++++++++++-------------
 2 files changed, 43 insertions(+), 25 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 86a1164..107df6c 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -1,4 +1,3 @@
-maxMemoryUsage=80;
 baseName="crunner0";
 namePrefix="cubs";
 runners="";
@@ -16,9 +15,9 @@ usedMemoryPercent;
 until [[ $memory -gt $maxMemoryUsage ]]; do
 	
 	runnerName="${namePrefix}${RANDOM}";
-	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d
-	if[[ $? -eq 0 ]] 
-	then
+	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
+	
+	if[[ "$?" -eq 0 ]]; then
 		runners="${runners};${runnerName}";
 	fi
 	usedMemoryPercent;
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index e7b96b3..9e7046d 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -167,6 +167,7 @@ var Worker = (function(){
 		var worker = this;
 		args.stopPercent = args.stopPercent || 80;
 		args.callback = args.callback || __empty;
+		args.errorCallback = args.errorCallback || __empty;
 
 		// dont make runners on out dated workers
 		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
@@ -180,23 +181,32 @@ var Worker = (function(){
 		}
 
 		worker.isBuildingRunners = true;
-		fs.read(__dirname + "../allocate_runners.sh", function(error, data){
-			console.log(data);
-			// lxc.exec(data, function(output){
-			// 	// output chould be list of runner names
-			// 	console.log(output);
-			// 	// for name in output:
-			// 	// var runner = Runner.create({
-			// 	// 	"name": name,
-			// 	// 	"worker": worker,
-			// 	// 	"label": worker.name + ':' + name
-			// 	// });
+		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
+			var command = `maxMemoryUsage=${args.stopPercent};\n${file.toString()}`;
+			lxc.exec(command, worker.ip, function(data, error, stderr){
+				// output chould be list of runner names
+				if(error){
+					// ugly
+					console.log("Error", worker.ip, error);
+					worker.isBuildingRunners = false;
+					args.errorCallback(error, worker, args);
+				} else {
+					console.log("exec:");
+					console.log(arguments);
+					var runners = data.split(";");
+					for (let idx = 0, stop = runners.length; idx < stop; idx++){
 
-			// 	// worker.availrunners.push(runner);
-			// 	// end for
-			// 	worker.isBuildingRunners = false;
-			// 	args.callback(worker);
-			// });
+						var runner = Runner.create({
+							"name": runners[idx],
+							"worker": worker,
+							"label": worker.name + ':' + runners[idx]
+						});
+						worker.availrunners.push(runner);
+					}
+					worker.isBuildingRunners = false;
+					args.callback(worker);
+				}
+			});
 		});
 	};
 
@@ -329,12 +339,21 @@ var WorkerCollection = (function(){
 		if(workers.length + workers.currentCreating >= workers.settings.max ) return false;
 		workers.currentCreating++;
 
+		var count = 0;
 		config = config || workers.settings;
-		Worker.initialize(function(worker){
-			console.log("Seeded runners on", worker.name);
-			workers.push(worker);
-			worker.register();
-			workers.currentCreating--;
+		Worker.initialize({
+			"callback": function(worker){
+				console.log("Seeded runners on", worker.name);
+				workers.push(worker);
+				worker.register();
+				workers.currentCreating--;
+			},
+			"errorCallback": function(error, worker, args){
+				if (count++ > 3){
+					args.errorCallback = function(){};
+				}
+				worker.newStartRunners(args);
+			}
 		}, config);
 	};
 
-- 
2.34.1


From 7bbf77c6ec5fa5c71adc7022317b84a7cc8dc48c Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 16 Oct 2017 19:11:47 -0400
Subject: [PATCH 25/42] fixed syntax error in allocate_runners.sh

---
 allocate_runners.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 107df6c..f906edc 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -17,10 +17,10 @@ until [[ $memory -gt $maxMemoryUsage ]]; do
 	runnerName="${namePrefix}${RANDOM}";
 	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
 	
-	if[[ "$?" -eq 0 ]]; then
-		runners="${runners};${runnerName}";
+	if [[ "$?" -eq 0 ]]; then
+		runners="${runnerName};${runners}";
 	fi
 	usedMemoryPercent;
 done
 
-echo $runners
\ No newline at end of file
+echo $runners;
\ No newline at end of file
-- 
2.34.1


From b385c0cb7f1ad9957bc500f908db6d1b40b5d120 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 16 Oct 2017 19:12:44 -0400
Subject: [PATCH 26/42] patch

---
 allocate_runners.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index f906edc..7ab5543 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -17,7 +17,7 @@ until [[ $memory -gt $maxMemoryUsage ]]; do
 	runnerName="${namePrefix}${RANDOM}";
 	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
 	
-	if [[ "$?" -eq 0 ]]; then
+	if [[ $? -eq 0 ]]; then
 		runners="${runnerName};${runners}";
 	fi
 	usedMemoryPercent;
-- 
2.34.1


From 178838366cdd06dfcca960cc3d77847ca5f8eb3d Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 16 Oct 2017 19:18:59 -0400
Subject: [PATCH 27/42] moving more code out of api.js

---
 routes/api.js               | 8 +++-----
 routes/worker_collection.js | 5 +++++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index 7e809b9..2cc177c 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -9,11 +9,9 @@ var doapi = require('../doapi')();
 
 var workers = require('./worker_collection.js');
 
-(function(){
-	console.log('========STARTING===========')
-	setInterval(workers.checkBalance, 15000);
-	workers.destroyByTag();
-})()
+console.log('========STARTING===========');
+
+workers.start();
 
 var attemptRun = function(req, res, runner, count){
 	count = count || 0;
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 9e7046d..ad1f61d 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -495,6 +495,11 @@ var WorkerCollection = (function(){
 
 	};
 
+	workers.start = function(interval){
+		setInterval(workers.checkBalance, interval || 15000);
+		workers.destroyByTag();
+	};
+
 	workers.settingsSave = function(){
 		// save the live settings file to disk
 
-- 
2.34.1


From 32a338437e23afb1c75d9bedd48bdb252e7a167e Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 16 Oct 2017 19:27:18 -0400
Subject: [PATCH 28/42] updated logic and names

---
 allocate_runners.sh         |  2 +-
 routes/worker_collection.js | 21 +++++++++++----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 7ab5543..6fe513f 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -18,7 +18,7 @@ until [[ $memory -gt $maxMemoryUsage ]]; do
 	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
 	
 	if [[ $? -eq 0 ]]; then
-		runners="${runnerName};${runners}";
+		runners="${runners};${runnerName}";
 	fi
 	usedMemoryPercent;
 done
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index ad1f61d..3bb62d4 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -143,7 +143,7 @@ var Worker = (function(){
 		});
 	};
 
-	proto.initialize = function(callback, config){
+	proto.initialize = function(hooks, config){
 		// Create droplet
 		// Once active the droplet begins to create runners
 		doapi.dropletToActive({
@@ -158,7 +158,7 @@ var Worker = (function(){
 			},
 			onActive: function(data, args){
 				var worker = Worker.create(data);
-				worker.newStartRunners(callback);
+				worker.newStartRunners(hooks);
 			}
 		});
 	};
@@ -195,13 +195,14 @@ var Worker = (function(){
 					console.log(arguments);
 					var runners = data.split(";");
 					for (let idx = 0, stop = runners.length; idx < stop; idx++){
-
-						var runner = Runner.create({
-							"name": runners[idx],
-							"worker": worker,
-							"label": worker.name + ':' + runners[idx]
-						});
-						worker.availrunners.push(runner);
+						if(runners[idx]){
+							var runner = Runner.create({
+								"name": runners[idx],
+								"worker": worker,
+								"label": worker.name + ':' + runners[idx]
+							});
+							worker.availrunners.push(runner);
+						}
 					}
 					worker.isBuildingRunners = false;
 					args.callback(worker);
@@ -211,7 +212,7 @@ var Worker = (function(){
 	};
 
 
-	proto.startRunners = function(args){
+	proto.__XstartRunners = function(args){
 		var worker = this;
 		
 		console.log('Starting runners on', worker.name, worker.ip);
-- 
2.34.1


From 5cd95e2843647c6010ae74da42744ee0a3037356 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 17 Oct 2017 11:19:35 -0400
Subject: [PATCH 29/42] adding  to bash script

---
 allocate_runners.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 6fe513f..bce00a0 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -23,4 +23,7 @@ until [[ $memory -gt $maxMemoryUsage ]]; do
 	usedMemoryPercent;
 done
 
-echo $runners;
\ No newline at end of file
+echo $runners;
+
+
+exit 0;
\ No newline at end of file
-- 
2.34.1


From 2fa3ae985b93691a8651aab27a3ee2b844b2e61c Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 18 Oct 2017 00:49:42 -0400
Subject: [PATCH 30/42] adding sync function

---
 allocate_runners.sh         |   3 +
 routes/worker_collection.js | 164 +++++++++++++++---------------------
 2 files changed, 72 insertions(+), 95 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index bce00a0..7bd38ce 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -1,3 +1,4 @@
+# maxMemoryUsage must be defined
 baseName="crunner0";
 namePrefix="cubs";
 runners="";
@@ -12,6 +13,8 @@ function usedMemoryPercent () {
 
 
 usedMemoryPercent;
+
+# maxMemoryUsage must be defined
 until [[ $memory -gt $maxMemoryUsage ]]; do
 	
 	runnerName="${namePrefix}${RANDOM}";
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 3bb62d4..6f71e29 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -58,7 +58,7 @@ var Runner = (function(){
 			runner.cleanUp();
 		}
 
-		runner.worker.newStartRunners();
+		runner.worker.startRunners();
 	};
 
 	proto.setTimeout = function(time){
@@ -95,8 +95,9 @@ var Worker = (function(){
 		worker.ip = worker.publicIP;
 		worker.usedrunners = 0;
 		worker.age = +(new Date());
-		worker.isBuildingRunners = false;
 		worker.canSchedule = true;
+		worker.isBuildingRunners = false;
+		worker.isSyncing = false;
 
 		return worker;
 	};
@@ -143,6 +144,59 @@ var Worker = (function(){
 		});
 	};
 
+	proto.sync = function(){
+		var worker = this;
+		worker.isSyncing = true;
+		// this will call the droplet or the droplet will send the data using a cron job
+
+		// mainly to update the active runners on the worker
+		// potentially collect stats about the droplet as well
+		// - check memory and check runners
+		// - when does start runners get called?
+
+		// worker.ramPercentUsed();
+
+
+		lxc.exec('lxc-ls --fancy', worker.ip, function(data, error, stderr){
+			if (error){
+				console.log("Sync Error: \n", error);
+			} else {
+				
+				var output = data.split("\n");
+				var keys = output.splice(0,1)[0].split(/\s+/).slice(0,-1);
+				var info = [];
+
+				keys = keys.map(function(v){return v.toLowerCase()});
+				output = output.slice(0).slice(0,-1);
+
+				for(var i in output){
+					if(output[i].match(/^-/)) continue; // compatibility with 1.x and 2.x output
+
+					var aIn = output[i].split(/\s+/).slice(0,-1);
+					var mapOut = {};
+					aIn.map(function(value,idx){
+						mapOut[keys[idx]] = value;
+					});
+					info.push(mapOut);
+					
+				}
+				worker.availrunners = [];
+
+				for (let idx = 0, stop = runners.length; idx < stop; idx++){
+					if(runners[idx].state !== "STOPPED"){
+						var runner = Runner.create({
+							"name": runners[idx].name,
+							"worker": worker,
+							"label": worker.name + ':' + runners[idx].name
+						});
+						worker.availrunners.push(runner);
+					}
+				}
+				worker.isBuildingRunners = false;
+			}
+		});
+	};
+
 	proto.initialize = function(hooks, config){
 		// Create droplet
 		// Once active the droplet begins to create runners
@@ -158,11 +212,12 @@ var Worker = (function(){
 			},
 			onActive: function(data, args){
 				var worker = Worker.create(data);
-				worker.newStartRunners(hooks);
+				worker.startRunners(hooks);
 			}
 		});
 	};
-	proto.newStartRunners = function(args){
+
+	proto.startRunners = function(args){
 		// onStart is not necessary
 		var worker = this;
 		args.stopPercent = args.stopPercent || 80;
@@ -182,101 +237,20 @@ var Worker = (function(){
 
 		worker.isBuildingRunners = true;
 		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
+			
 			var command = `maxMemoryUsage=${args.stopPercent};\n${file.toString()}`;
+			
 			lxc.exec(command, worker.ip, function(data, error, stderr){
-				// output chould be list of runner names
-				if(error){
-					// ugly
-					console.log("Error", worker.ip, error);
-					worker.isBuildingRunners = false;
-					args.errorCallback(error, worker, args);
-				} else {
-					console.log("exec:");
-					console.log(arguments);
-					var runners = data.split(";");
-					for (let idx = 0, stop = runners.length; idx < stop; idx++){
-						if(runners[idx]){
-							var runner = Runner.create({
-								"name": runners[idx],
-								"worker": worker,
-								"label": worker.name + ':' + runners[idx]
-							});
-							worker.availrunners.push(runner);
-						}
-					}
-					worker.isBuildingRunners = false;
+				
+				// Just send the job over and set a timeout 
+				// to wait before checking runners
+				setTimeout(function(){
+					worker.sync();
 					args.callback(worker);
-				}
+				}, 10000);
+
 			});
-		});
-	};
 
-
-	proto.__XstartRunners = function(args){
-		var worker = this;
-		
-		console.log('Starting runners on', worker.name, worker.ip);
-		// dont make runners on out dated workers
-		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
-			if(worker) console.log(`Blocked worker(${worker.image.id}), current image ${worker.settings.image}. Building: ${worker.isBuildingRunners}`)
-			return ;
-		}
-		args = args || {};
-		// percent of used RAM to stop runner creation
-		args.stopPercent = args.stopPercent || 80;
-		args.onStart = args.onStart || __empty;
-		args.onDone = args.onDone || __empty;
-
-		worker.isBuildingRunners = true;
-		
-		// no longer needed
-		worker.ramPercentUsed(function(usedMemPercent){
-			console.log(arguments);
-			if(usedMemPercent > args.stopPercent ){
-				worker.isBuildingRunners = false;
-				console.log('---using', String(usedMemPercent).trim(),
-					'percent memory, stopping runner creation!', worker.availrunners.length, 
-					'created on ', worker.name
-				);
-				args.onDone(worker, args);
-				return ;
-			} else if (usedMemPercent !== ""){
-				console.log('+++using', String(usedMemPercent).trim(),
-					'percent memory on ', worker.name, 
-					'Runners:', worker.availrunners.length,
-					`Used? ${usedMemPercent}`
-
-				);
-				var name = 'crunner-' + utils.uuid();
-				// console.log('Free ram check passed!')
-				lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
-					setTimeout(function(){
-						worker.isBuildingRunners = false;
-						worker.startRunners(args);
-					}, 0);
-
-					if(data.ip){
-						console.log('started runner on', worker.name);
-
-						var runner = Runner.create({
-							"ip": data.ip,
-							"name": name,
-							"worker": worker,
-							"label": worker.name + ':' + name
-						});
-
-						args.onStart(worker, args);
-
-						worker.availrunners.push(runner);
-					}
-
-				});
-			} else {
-				return setTimeout(function(){
-					worker.isBuildingRunners = false;
-					worker.startRunners(args);
-				}, 0);
-			}
 		});
 	};
 
@@ -353,7 +327,7 @@ var WorkerCollection = (function(){
 				if (count++ > 3){
 					args.errorCallback = function(){};
 				}
-				worker.newStartRunners(args);
+				worker.startRunners(args);
 			}
 		}, config);
 	};
-- 
2.34.1


From 36db6be4bc5458a1b88c2958fcb4b2380621808f Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 18 Oct 2017 00:59:47 -0400
Subject: [PATCH 31/42] Adding some logging."

---
 routes/worker_collection.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 6f71e29..fac83b9 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -144,9 +144,10 @@ var Worker = (function(){
 		});
 	};
 
-	proto.sync = function(){
+	proto.sync = function(callback){
 		var worker = this;
 		worker.isSyncing = true;
+		callback = callback || __empty;
 		// this will call the droplet or the droplet will send the data using a cron job
 
 		// mainly to update the active runners on the worker
@@ -180,6 +181,8 @@ var Worker = (function(){
 					info.push(mapOut);
 					
 				}
+				console.log(`RUNNERS FOUND[=> ${worker.ip}`);
+				console.log(`RUNNERS FOUND[=> ${info}`);
 				worker.availrunners = [];
 
 				for (let idx = 0, stop = runners.length; idx < stop; idx++){
@@ -193,6 +196,8 @@ var Worker = (function(){
 					}
 				}
 				worker.isBuildingRunners = false;
+				console.log(`RUNNERS AVAILABLE[=> ${worker.availrunners}`);
+				callback(worker);
 			}
 		});
 	};
@@ -245,8 +250,7 @@ var Worker = (function(){
 				// Just send the job over and set a timeout 
 				// to wait before checking runners
 				setTimeout(function(){
-					worker.sync();
-					args.callback(worker);
+					worker.sync(args.callback);
 				}, 10000);
 
 			});
-- 
2.34.1


From 0ba6589aa789e4ad9bec8826412daf045957915d Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 18 Oct 2017 01:00:55 -0400
Subject: [PATCH 32/42] fixed runner

---
 routes/worker_collection.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index fac83b9..5b26e86 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -165,7 +165,7 @@ var Worker = (function(){
 				
 				var output = data.split("\n");
 				var keys = output.splice(0,1)[0].split(/\s+/).slice(0,-1);
-				var info = [];
+				var runners = [];
 
 				keys = keys.map(function(v){return v.toLowerCase()});
 				output = output.slice(0).slice(0,-1);
@@ -178,7 +178,7 @@ var Worker = (function(){
 					aIn.map(function(value,idx){
 						mapOut[keys[idx]] = value;
 					});
-					info.push(mapOut);
+					runners.push(mapOut);
 					
 				}
 				console.log(`RUNNERS FOUND[=> ${worker.ip}`);
-- 
2.34.1


From f8174428a112a4786978c51e67c5a93c24e1a649 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 18 Oct 2017 01:04:16 -0400
Subject: [PATCH 33/42] patch

---
 routes/worker_collection.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 5b26e86..e32e785 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -182,7 +182,7 @@ var Worker = (function(){
 					
 				}
 				console.log(`RUNNERS FOUND[=> ${worker.ip}`);
-				console.log(`RUNNERS FOUND[=> ${info}`);
+				console.log(`RUNNERS FOUND[=> ${runners}`);
 				worker.availrunners = [];
 
 				for (let idx = 0, stop = runners.length; idx < stop; idx++){
-- 
2.34.1


From 02e7869846ac7db510186406f377ddc0311d2cdf Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Wed, 18 Oct 2017 01:10:32 -0400
Subject: [PATCH 34/42] patch

---
 routes/worker_collection.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index e32e785..98fa7f5 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -182,7 +182,7 @@ var Worker = (function(){
 					
 				}
 				console.log(`RUNNERS FOUND[=> ${worker.ip}`);
-				console.log(`RUNNERS FOUND[=> ${runners}`);
+				console.log(`RUNNERS FOUND[=>`, runners);
 				worker.availrunners = [];
 
 				for (let idx = 0, stop = runners.length; idx < stop; idx++){
@@ -196,7 +196,7 @@ var Worker = (function(){
 					}
 				}
 				worker.isBuildingRunners = false;
-				console.log(`RUNNERS AVAILABLE[=> ${worker.availrunners}`);
+				console.log(`RUNNERS AVAILABLE[=>`, worker.availrunners);
 				callback(worker);
 			}
 		});
-- 
2.34.1


From e2371fe44ec764d06407df8b6172c7e2cbbea989 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Thu, 19 Oct 2017 12:45:11 -0400
Subject: [PATCH 35/42] updating command for startRunners to create cron job.

---
 allocate_runners.sh         | 29 +++++++++++++++--------------
 routes/worker_collection.js | 11 ++++++-----
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 7bd38ce..6f21d2a 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -11,22 +11,23 @@ function usedMemoryPercent () {
 	memory=$(expr $difference / $totalMemory);
 }
 
+function buildRunners () {
 
-usedMemoryPercent;
-
-# maxMemoryUsage must be defined
-until [[ $memory -gt $maxMemoryUsage ]]; do
-	
-	runnerName="${namePrefix}${RANDOM}";
-	lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
-	
-	if [[ $? -eq 0 ]]; then
-		runners="${runners};${runnerName}";
-	fi
 	usedMemoryPercent;
-done
-
-echo $runners;
 
+	# maxMemoryUsage must be defined
+	until [[ $memory -gt $maxMemoryUsage ]]; do
+		
+		runnerName="${namePrefix}${RANDOM}";
+		lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
+		
+		if [[ $? -eq 0 ]]; then
+			runners="${runners};${runnerName}";
+		fi
+		usedMemoryPercent;
+	done
 
+	echo $runners;
+}
+buildRunners;
 exit 0;
\ No newline at end of file
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 98fa7f5..2a339bc 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -242,16 +242,17 @@ var Worker = (function(){
 
 		worker.isBuildingRunners = true;
 		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
-			
-			var command = `maxMemoryUsage=${args.stopPercent};\n${file.toString()}`;
-			
+			var command = '(crontab -l && echo "1 * * * * export PATH=${PATH};' + `export maxMemoryUsage=${args.stopPercent}; echo '${file.toString("base64")}'|base64 --decode|bash") | crontab -`;
+			// COMMAND:  (crontab -l && echo "1 * * * * export PATH=${PATH};export maxMemoryUsage=80; echo 'IyBtYXhNZW1vcnlVc2FnZSBtdXN0IGJlIGRlZmluZWQKYmFzZU5hbWU9ImNydW5uZXIwIjsKbmFtZVByZWZpeD0iY3VicyI7CnJ1bm5lcnM9IiI7CgpmdW5jdGlvbiB1c2VkTWVtb3J5UGVyY2VudCAoKSB7CgltZW1vcnlBdmFpbGFibGU9JChoZWFkIC9wcm9jL21lbWluZm98Z3JlcCBNZW1BdmFpbHxncmVwIC1QbyAnXGQrJyk7Cgl0b3RhbE1lbW9yeT0kKGhlYWQgL3Byb2MvbWVtaW5mb3xncmVwIE1lbVRvdGFsfGdyZXAgLVBvICdcZCsnKTsKCWRpZmZlcmVuY2U9JChleHByICR0b3RhbE1lbW9yeSAtICRtZW1vcnlBdmFpbGFibGUpOwoJZGlmZmVyZW5jZT0kKGV4cHIgJGRpZmZlcmVuY2UgXCogMTAwKTsKCW1lbW9yeT0kKGV4cHIgJGRpZmZlcmVuY2UgLyAkdG90YWxNZW1vcnkpOwp9CgpmdW5jdGlvbiBidWlsZFJ1bm5lcnMgKCkgewoKCXVzZWRNZW1vcnlQZXJjZW50OwoKCSMgbWF4TWVtb3J5VXNhZ2UgbXVzdCBiZSBkZWZpbmVkCgl1bnRpbCBbWyAkbWVtb3J5IC1ndCAkbWF4TWVtb3J5VXNhZ2UgXV07IGRvCgkJCgkJcnVubmVyTmFtZT0iJHtuYW1lUHJlZml4fSR7UkFORE9NfSI7CgkJbHhjLXN0YXJ0LWVwaGVtZXJhbCAtbyAkYmFzZU5hbWUgLW4gJHJ1bm5lck5hbWUgLS11bmlvbi10eXBlIG92ZXJsYXlmcyAtZDsKCQkKCQlpZiBbWyAkPyAtZXEgMCBdXTsgdGhlbgoJCQlydW5uZXJzPSIke3J1bm5lcnN9OyR7cnVubmVyTmFtZX0iOwoJCWZpCgkJdXNlZE1lbW9yeVBlcmNlbnQ7Cglkb25lCgoJZWNobyAkcnVubmVyczsKfQpidWlsZFJ1bm5lcnM7CmV4aXQgMDs='|base64 --decode|bash") | crontab -
+			// RESULT IN SHELL: no crontab for virt
 			lxc.exec(command, worker.ip, function(data, error, stderr){
-				
+				console.log(arguments);
 				// Just send the job over and set a timeout 
 				// to wait before checking runners
 				setTimeout(function(){
 					worker.sync(args.callback);
-				}, 10000);
+					// 900000 < thats 15 min not 90 sec
+				}, 90000);
 
 			});
 
-- 
2.34.1


From 7bfda7be236ff08ee447c9ed5e57860890367667 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Thu, 19 Oct 2017 23:08:14 -0400
Subject: [PATCH 36/42] tweaking crontab creation.

---
 routes/worker_collection.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 2a339bc..23f4efd 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -242,9 +242,9 @@ var Worker = (function(){
 
 		worker.isBuildingRunners = true;
 		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
-			var command = '(crontab -l && echo "1 * * * * export PATH=${PATH};' + `export maxMemoryUsage=${args.stopPercent}; echo '${file.toString("base64")}'|base64 --decode|bash") | crontab -`;
-			// COMMAND:  (crontab -l && echo "1 * * * * export PATH=${PATH};export maxMemoryUsage=80; echo 'IyBtYXhNZW1vcnlVc2FnZSBtdXN0IGJlIGRlZmluZWQKYmFzZU5hbWU9ImNydW5uZXIwIjsKbmFtZVByZWZpeD0iY3VicyI7CnJ1bm5lcnM9IiI7CgpmdW5jdGlvbiB1c2VkTWVtb3J5UGVyY2VudCAoKSB7CgltZW1vcnlBdmFpbGFibGU9JChoZWFkIC9wcm9jL21lbWluZm98Z3JlcCBNZW1BdmFpbHxncmVwIC1QbyAnXGQrJyk7Cgl0b3RhbE1lbW9yeT0kKGhlYWQgL3Byb2MvbWVtaW5mb3xncmVwIE1lbVRvdGFsfGdyZXAgLVBvICdcZCsnKTsKCWRpZmZlcmVuY2U9JChleHByICR0b3RhbE1lbW9yeSAtICRtZW1vcnlBdmFpbGFibGUpOwoJZGlmZmVyZW5jZT0kKGV4cHIgJGRpZmZlcmVuY2UgXCogMTAwKTsKCW1lbW9yeT0kKGV4cHIgJGRpZmZlcmVuY2UgLyAkdG90YWxNZW1vcnkpOwp9CgpmdW5jdGlvbiBidWlsZFJ1bm5lcnMgKCkgewoKCXVzZWRNZW1vcnlQZXJjZW50OwoKCSMgbWF4TWVtb3J5VXNhZ2UgbXVzdCBiZSBkZWZpbmVkCgl1bnRpbCBbWyAkbWVtb3J5IC1ndCAkbWF4TWVtb3J5VXNhZ2UgXV07IGRvCgkJCgkJcnVubmVyTmFtZT0iJHtuYW1lUHJlZml4fSR7UkFORE9NfSI7CgkJbHhjLXN0YXJ0LWVwaGVtZXJhbCAtbyAkYmFzZU5hbWUgLW4gJHJ1bm5lck5hbWUgLS11bmlvbi10eXBlIG92ZXJsYXlmcyAtZDsKCQkKCQlpZiBbWyAkPyAtZXEgMCBdXTsgdGhlbgoJCQlydW5uZXJzPSIke3J1bm5lcnN9OyR7cnVubmVyTmFtZX0iOwoJCWZpCgkJdXNlZE1lbW9yeVBlcmNlbnQ7Cglkb25lCgoJZWNobyAkcnVubmVyczsKfQpidWlsZFJ1bm5lcnM7CmV4aXQgMDs='|base64 --decode|bash") | crontab -
-			// RESULT IN SHELL: no crontab for virt
+			var command = '(crontab -l > /tmp/virtjobs && echo "1 * * * * export PATH=${PATH};' +
+			"export maxMemoryUsage=${args.stopPercent};" + `echo '${file.toString("base64")}'|base64 --decode|bash")` 
+			+ ">> /tmp/virtjobs && crontab /tmp/virtjobs && rm /tmp/virtjobs;";
 			lxc.exec(command, worker.ip, function(data, error, stderr){
 				console.log(arguments);
 				// Just send the job over and set a timeout 
-- 
2.34.1


From 734eaeac2f001e6881b8e4d7678c29224018e25f Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Sat, 21 Oct 2017 11:05:04 -0400
Subject: [PATCH 37/42] adjusting cronjob

---
 allocate_runners.sh         |  9 +++------
 routes/crontab_syslog.txt   | 25 +++++++++++++++++++++++++
 routes/worker_collection.js | 26 ++++++++++++++++++++------
 3 files changed, 48 insertions(+), 12 deletions(-)
 create mode 100644 routes/crontab_syslog.txt

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 6f21d2a..a26ed4b 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -1,7 +1,4 @@
 # maxMemoryUsage must be defined
-baseName="crunner0";
-namePrefix="cubs";
-runners="";
 
 function usedMemoryPercent () {
 	memoryAvailable=$(head /proc/meminfo|grep MemAvail|grep -Po '\d+');
@@ -12,7 +9,9 @@ function usedMemoryPercent () {
 }
 
 function buildRunners () {
-
+	baseName="crunner0";
+	namePrefix="cubs";
+	runners="";
 	usedMemoryPercent;
 
 	# maxMemoryUsage must be defined
@@ -26,8 +25,6 @@ function buildRunners () {
 		fi
 		usedMemoryPercent;
 	done
-
-	echo $runners;
 }
 buildRunners;
 exit 0;
\ No newline at end of file
diff --git a/routes/crontab_syslog.txt b/routes/crontab_syslog.txt
new file mode 100644
index 0000000..1b9b855
--- /dev/null
+++ b/routes/crontab_syslog.txt
@@ -0,0 +1,25 @@
+Oct 20 21:52:16 dev--CLW10-3066 crontab[1885]: (virt) REPLACE (virt)
+Oct 20 21:52:37 dev--CLW10-3066 crontab[2166]: (virt) LIST (virt)
+Oct 20 21:53:03 dev--CLW10-3066 crontab[2168]: (virt) LIST (virt)
+Oct 20 22:01:01 dev--CLW10-3066 CRON[18873]: (virt) CMD (source /home/virt/allocate_runners.sh)
+Oct 20 23:01:01 dev--CLW10-3066 CRON[1867]: (virt) CMD (source /home/virt/allocate_runners.sh)
+
+
+
+
+ALL CRON
+Oct 20 21:52:13 dev--CLW10-3066 cron[1443]: (CRON) INFO (pidfile fd = 3)
+Oct 20 21:52:13 dev--CLW10-3066 cron[1443]: (CRON) INFO (Running @reboot jobs)
+Oct 20 21:52:16 dev--CLW10-3066 crontab[1885]: (virt) REPLACE (virt)
+Oct 20 21:52:37 dev--CLW10-3066 crontab[2166]: (virt) LIST (virt)
+Oct 20 21:53:03 dev--CLW10-3066 crontab[2168]: (virt) LIST (virt)
+Oct 20 22:01:01 dev--CLW10-3066 CRON[18873]: (virt) CMD (source /home/virt/allocate_runners.sh)
+Oct 20 22:01:01 dev--CLW10-3066 CRON[18870]: (CRON) info (No MTA installed, discarding output)
+Oct 20 22:17:01 dev--CLW10-3066 cron[1443]: Authentication token is no longer valid; new one required
+Oct 20 22:17:01 dev--CLW10-3066 CRON[18913]: Authentication token is no longer valid; new one required
+Oct 20 22:17:53 dev--CLW10-3066 cron[1393]: (CRON) INFO (pidfile fd = 3)
+Oct 20 22:17:53 dev--CLW10-3066 cron[1393]: (CRON) INFO (Running @reboot jobs)
+Oct 20 23:01:01 dev--CLW10-3066 CRON[1867]: (virt) CMD (source /home/virt/allocate_runners.sh)
+Oct 20 23:01:01 dev--CLW10-3066 CRON[1865]: (CRON) info (No MTA installed, discarding output)
+Oct 20 23:17:01 dev--CLW10-3066 cron[1393]: Authentication token is no longer valid; new one required
+Oct 20 23:17:01 dev--CLW10-3066 CRON[1897]: Authentication token is no longer valid; new one required
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 23f4efd..2b786be 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -222,7 +222,15 @@ var Worker = (function(){
 		});
 	};
 
-	proto.startRunners = function(args){
+	proto.__buildCommand = function(file, maxMemoryUsage){
+		var command = `echo "export PATH=\${PATH};export maxMemoryUsage=${maxMemoryUsage};`;
+		command += `echo '${file.toString("base64")}'|base64 --decode|bash" | cat > /home/virt/allocate_runners.sh`;
+		command += ` && echo "*/1 * * * * source /home/virt/allocate_runners.sh >> /home/virt/allocate_runners.log 2>&1" | crontab -;`;
+		return command;
+	};
+
+	proto.startRunners = function(args, count){
+		if ( count > 3) return;
 		// onStart is not necessary
 		var worker = this;
 		args.stopPercent = args.stopPercent || 80;
@@ -242,18 +250,24 @@ var Worker = (function(){
 
 		worker.isBuildingRunners = true;
 		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
-			var command = '(crontab -l > /tmp/virtjobs && echo "1 * * * * export PATH=${PATH};' +
-			"export maxMemoryUsage=${args.stopPercent};" + `echo '${file.toString("base64")}'|base64 --decode|bash")` 
-			+ ">> /tmp/virtjobs && crontab /tmp/virtjobs && rm /tmp/virtjobs;";
+			var command = worker.__buildCommand(file, args.stopPercent);
+				
 			lxc.exec(command, worker.ip, function(data, error, stderr){
-				console.log(arguments);
+				if (error) { 
+					console.log("ERROR: ",stderr);
+					setTimeout(function(){
+						count = count || 0;
+						worker.startRunners(args, ++count);
+					}, 0);
+				} else {
+					console.log("SUCCESS: ", data , worker.ip);
+				}
 				// Just send the job over and set a timeout 
 				// to wait before checking runners
 				setTimeout(function(){
 					worker.sync(args.callback);
 					// 900000 < thats 15 min not 90 sec
 				}, 90000);
-
 			});
 
 		});
-- 
2.34.1


From 0a4162400b613bed0aecb15f822de6b6a999d29b Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Mon, 23 Oct 2017 15:53:49 -0400
Subject: [PATCH 38/42] updating initialize function to include user_data

---
 allocate_runners.sh         |  6 ++++--
 routes/crontab_syslog.txt   | 25 -------------------------
 routes/worker_collection.js |  5 +++--
 3 files changed, 7 insertions(+), 29 deletions(-)
 delete mode 100644 routes/crontab_syslog.txt

diff --git a/allocate_runners.sh b/allocate_runners.sh
index a26ed4b..15f90b0 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -10,14 +10,14 @@ function usedMemoryPercent () {
 
 function buildRunners () {
 	baseName="crunner0";
-	namePrefix="cubs";
+	namePrefix="crunner-batch-${RANDOM}";
 	runners="";
 	usedMemoryPercent;
 
 	# maxMemoryUsage must be defined
 	until [[ $memory -gt $maxMemoryUsage ]]; do
 		
-		runnerName="${namePrefix}${RANDOM}";
+		runnerName="${namePrefix}-id-${RANDOM}";
 		lxc-start-ephemeral -o $baseName -n $runnerName --union-type overlayfs -d;
 		
 		if [[ $? -eq 0 ]]; then
@@ -27,4 +27,6 @@ function buildRunners () {
 	done
 }
 buildRunners;
+
+# Add curl to manager here. Sending status report to manager
 exit 0;
\ No newline at end of file
diff --git a/routes/crontab_syslog.txt b/routes/crontab_syslog.txt
deleted file mode 100644
index 1b9b855..0000000
--- a/routes/crontab_syslog.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-Oct 20 21:52:16 dev--CLW10-3066 crontab[1885]: (virt) REPLACE (virt)
-Oct 20 21:52:37 dev--CLW10-3066 crontab[2166]: (virt) LIST (virt)
-Oct 20 21:53:03 dev--CLW10-3066 crontab[2168]: (virt) LIST (virt)
-Oct 20 22:01:01 dev--CLW10-3066 CRON[18873]: (virt) CMD (source /home/virt/allocate_runners.sh)
-Oct 20 23:01:01 dev--CLW10-3066 CRON[1867]: (virt) CMD (source /home/virt/allocate_runners.sh)
-
-
-
-
-ALL CRON
-Oct 20 21:52:13 dev--CLW10-3066 cron[1443]: (CRON) INFO (pidfile fd = 3)
-Oct 20 21:52:13 dev--CLW10-3066 cron[1443]: (CRON) INFO (Running @reboot jobs)
-Oct 20 21:52:16 dev--CLW10-3066 crontab[1885]: (virt) REPLACE (virt)
-Oct 20 21:52:37 dev--CLW10-3066 crontab[2166]: (virt) LIST (virt)
-Oct 20 21:53:03 dev--CLW10-3066 crontab[2168]: (virt) LIST (virt)
-Oct 20 22:01:01 dev--CLW10-3066 CRON[18873]: (virt) CMD (source /home/virt/allocate_runners.sh)
-Oct 20 22:01:01 dev--CLW10-3066 CRON[18870]: (CRON) info (No MTA installed, discarding output)
-Oct 20 22:17:01 dev--CLW10-3066 cron[1443]: Authentication token is no longer valid; new one required
-Oct 20 22:17:01 dev--CLW10-3066 CRON[18913]: Authentication token is no longer valid; new one required
-Oct 20 22:17:53 dev--CLW10-3066 cron[1393]: (CRON) INFO (pidfile fd = 3)
-Oct 20 22:17:53 dev--CLW10-3066 cron[1393]: (CRON) INFO (Running @reboot jobs)
-Oct 20 23:01:01 dev--CLW10-3066 CRON[1867]: (virt) CMD (source /home/virt/allocate_runners.sh)
-Oct 20 23:01:01 dev--CLW10-3066 CRON[1865]: (CRON) info (No MTA installed, discarding output)
-Oct 20 23:17:01 dev--CLW10-3066 cron[1393]: Authentication token is no longer valid; new one required
-Oct 20 23:17:01 dev--CLW10-3066 CRON[1897]: Authentication token is no longer valid; new one required
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 2b786be..37deab5 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -209,6 +209,7 @@ var Worker = (function(){
 			name: config.tagPrefix + (config.version+"") + '-' + utils.uuid(),
 			image: config.image,
 			size: config.size,
+			user_data: proto.__buildCommand(config.maxMemoryUsage || 80),
 			onCreate: function(data){
 				doapi.dropletSetTag(
 					config.tagPrefix + config.version, 
@@ -224,8 +225,8 @@ var Worker = (function(){
 
 	proto.__buildCommand = function(file, maxMemoryUsage){
 		var command = `echo "export PATH=\${PATH};export maxMemoryUsage=${maxMemoryUsage};`;
-		command += `echo '${file.toString("base64")}'|base64 --decode|bash" | cat > /home/virt/allocate_runners.sh`;
-		command += ` && echo "*/1 * * * * source /home/virt/allocate_runners.sh >> /home/virt/allocate_runners.log 2>&1" | crontab -;`;
+		command += `echo '${file.toString("base64")}'|base64 --decode|bash" | cat > /home/virt/allocate_runners.sh && chmod virt +x /home/virt/allocate_runners.sh`;
+		command += ` && echo "*/1 * * * * /home/virt/allocate_runners.sh >> /home/virt/allocate_runners.log 2>&1" | crontab -u virt -;`;
 		return command;
 	};
 
-- 
2.34.1


From 3db02e036e6b303fe73f5df29ec0454cd3854bd8 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 24 Oct 2017 12:03:29 -0400
Subject: [PATCH 39/42] updating runner initialization to send user_data script
 to setup workers cronjob.

---
 allocate_runners.sh         |   1 +
 routes/worker_collection.js | 105 +++++++++++++++++++++++++-----------
 2 files changed, 76 insertions(+), 30 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 15f90b0..5897cb4 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -29,4 +29,5 @@ function buildRunners () {
 buildRunners;
 
 # Add curl to manager here. Sending status report to manager
+curl -X POST "${PHONE_HOME}" -d'{"memory":${memory}, "runnersShort": "${runners}", "runnersLong": "'"$(lxc-ls --fancy)"'", "id": "${WORKER_UUID}"}' 
 exit 0;
\ No newline at end of file
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 37deab5..be88648 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -57,8 +57,8 @@ var Runner = (function(){
 		if(runner.hasOwnProperty('cleanUp')){
 			runner.cleanUp();
 		}
-
-		runner.worker.startRunners();
+		// TODO: Determine if this call is even needed
+		runner.worker.sync();
 	};
 
 	proto.setTimeout = function(time){
@@ -84,6 +84,8 @@ var Worker = (function(){
 	// settings should probably be retrieved via a function 
 	proto.settings = settings;
 
+	maxSyncAttempts = 3;
+
 	proto.create = function(config){
 		var worker = Object.create(proto);
 		Object.assign(worker, config);
@@ -98,6 +100,8 @@ var Worker = (function(){
 		worker.canSchedule = true;
 		worker.isBuildingRunners = false;
 		worker.isSyncing = false;
+		worker.syncAttempts = 0;
+
 
 		return worker;
 	};
@@ -144,10 +148,15 @@ var Worker = (function(){
 		});
 	};
 
-	proto.sync = function(callback){
+
+	// When should this be called
+	proto.sync = function(maxAttempts, callback, errorCallback){
+		maxAttempts = maxAttempts || maxSyncAttempts;
+
 		var worker = this;
 		worker.isSyncing = true;
 		callback = callback || __empty;
+		errorCallback = errorCallback || __empty;
 		// this will call the droplet or the droplet will send the data using a cron job
 
 		// mainly to update the active runners on the worker
@@ -161,6 +170,17 @@ var Worker = (function(){
 		lxc.exec('lxc-ls --fancy', worker.ip, function(data, error, stderr){
 			if (error){
 				console.log("Sync Error: \n", error);
+				if (worker.syncAttempts > maxAttempts){
+					setTimeout(function(){
+						errorCallback(error, worker);
+					}, 0);
+				} else {
+					console.log("Waiting 30 secongs")
+					worker.syncAttempts++;
+					setTimeout(function(){
+						worker.sync(maxAttempts, callback, errorCallback);
+					}, 30000);
+				}
 			} else {
 				
 				var output = data.split("\n");
@@ -195,39 +215,67 @@ var Worker = (function(){
 						worker.availrunners.push(runner);
 					}
 				}
-				worker.isBuildingRunners = false;
 				console.log(`RUNNERS AVAILABLE[=>`, worker.availrunners);
+				// TODO: Determine if this flag is needed anymore
+				worker.isBuildingRunners = false;
+				worker.isSyncing = false;
+				worker.syncAttempts = 0;
 				callback(worker);
 			}
 		});
 	};
 
-	proto.initialize = function(hooks, config){
+	proto.initialize = function(args, config){
 		// Create droplet
 		// Once active the droplet begins to create runners
-		doapi.dropletToActive({
-			name: config.tagPrefix + (config.version+"") + '-' + utils.uuid(),
-			image: config.image,
-			size: config.size,
-			user_data: proto.__buildCommand(config.maxMemoryUsage || 80),
-			onCreate: function(data){
-				doapi.dropletSetTag(
-					config.tagPrefix + config.version, 
-					data.droplet.id
-				);
-			},
-			onActive: function(data, args){
-				var worker = Worker.create(data);
-				worker.startRunners(hooks);
-			}
+		var maxMemoryUsage = args.maxMemoryUsage || config.maxMemoryUsage || 80;
+		var worker_uuid = utils.uuid();
+		// var phone_home = config.home || "/worker/ping";
+
+		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
+
+			doapi.dropletToActive({
+				name: config.tagPrefix + (config.version + "") + '-' + utils.uuid(),
+				image: config.image,
+				size: config.size,
+				user_data: proto.__buildCommand(file, maxMemoryUsage, worker_uuid, phone_home),
+				
+				onCreate: function(data){
+					doapi.dropletSetTag(
+						config.tagPrefix + config.version, 
+						data.droplet.id
+					);
+				},
+				onActive: function(data, args){
+					data.worker_uuid = worker_uuid;
+					var worker = Worker.create(data);
+					
+					// Just send the job over and set a timeout 
+					// to wait before checking runners
+					setTimeout(function(){
+						worker.sync(args.callback);
+					}, 90000);
+				}
+			});
 		});
 	};
 
-	proto.__buildCommand = function(file, maxMemoryUsage){
-		var command = `echo "export PATH=\${PATH};export maxMemoryUsage=${maxMemoryUsage};`;
-		command += `echo '${file.toString("base64")}'|base64 --decode|bash" | cat > /home/virt/allocate_runners.sh && chmod virt +x /home/virt/allocate_runners.sh`;
-		command += ` && echo "*/1 * * * * /home/virt/allocate_runners.sh >> /home/virt/allocate_runners.log 2>&1" | crontab -u virt -;`;
-		return command;
+	proto.__buildCommand = function(file, maxMemoryUsage, worker_uuid, phone_home){
+		var scriptSetup, script, createScript, makeScriptExecutable, setupCrontab;
+		var interval = 1;
+		
+		// worker_uuid and phone_home are only usable with localhost tunnels setup in dev
+		// cronjobSetup = `export PATH=\${PATH};export WORKER_UUID="${worker_uuid}";export PHONE_HOME=${phone_home};export maxMemoryUsage=${maxMemoryUsage};`;
+		scriptSetup = `export PATH=\${PATH};export WORKER_UUID="${worker_uuid}";export maxMemoryUsage=${maxMemoryUsage};`;
+		script = scriptSetup + `echo '${file.toString("base64")}'|base64 --decode|bash`;
+		
+		createScript = `echo "${script}" | cat > /home/virt/allocate_runners.sh`;
+		
+		makeScriptExecutable = `chmod virt +x /home/virt/allocate_runners.sh`;
+		
+		setupCrontab = `echo "*/${interval} * * * * /home/virt/allocate_runners.sh > /home/virt/allocate_runners.log 2>&1" | crontab -u virt -`;
+		
+		return `${createScript} && ${makeScriptExecutable} && ${setupCrontab};`;
 	};
 
 	proto.startRunners = function(args, count){
@@ -343,11 +391,8 @@ var WorkerCollection = (function(){
 				worker.register();
 				workers.currentCreating--;
 			},
-			"errorCallback": function(error, worker, args){
-				if (count++ > 3){
-					args.errorCallback = function(){};
-				}
-				worker.startRunners(args);
+			"errorCallback": function(error, worker){
+				// destroy worker
 			}
 		}, config);
 	};
-- 
2.34.1


From db8914ed7f56b314b5931233f30a7f13c8ba696d Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 24 Oct 2017 14:45:03 -0400
Subject: [PATCH 40/42] Updating Worker.sync and WorkerCollection.checkBalance

- adding maxAttempts and errorCallback to Worker.sync
- split checkBalance into two functions
- made checkZombies async so it can utilize Worker.sync
---
 allocate_runners.sh         |  2 +-
 routes/worker_collection.js | 65 ++++++++++++++++++++++---------------
 2 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/allocate_runners.sh b/allocate_runners.sh
index 5897cb4..79e018c 100644
--- a/allocate_runners.sh
+++ b/allocate_runners.sh
@@ -29,5 +29,5 @@ function buildRunners () {
 buildRunners;
 
 # Add curl to manager here. Sending status report to manager
-curl -X POST "${PHONE_HOME}" -d'{"memory":${memory}, "runnersShort": "${runners}", "runnersLong": "'"$(lxc-ls --fancy)"'", "id": "${WORKER_UUID}"}' 
+# curl -X POST "${PHONE_HOME}" -d'{"memory":${memory}, "runnersShort": "${runners}", "runnersLong": "'"$(lxc-ls --fancy)"'", "id": "${WORKER_UUID}"}' 
 exit 0;
\ No newline at end of file
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index be88648..5890bef 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -58,7 +58,7 @@ var Runner = (function(){
 			runner.cleanUp();
 		}
 		// TODO: Determine if this call is even needed
-		runner.worker.sync();
+		// runner.worker.sync();
 	};
 
 	proto.setTimeout = function(time){
@@ -84,7 +84,7 @@ var Worker = (function(){
 	// settings should probably be retrieved via a function 
 	proto.settings = settings;
 
-	maxSyncAttempts = 3;
+	var maxSyncAttempts = 6;
 
 	proto.create = function(config){
 		var worker = Object.create(proto);
@@ -150,7 +150,7 @@ var Worker = (function(){
 
 
 	// When should this be called
-	proto.sync = function(maxAttempts, callback, errorCallback){
+	proto.sync = function(callback, errorCallback, maxAttempts){
 		maxAttempts = maxAttempts || maxSyncAttempts;
 
 		var worker = this;
@@ -175,11 +175,11 @@ var Worker = (function(){
 						errorCallback(error, worker);
 					}, 0);
 				} else {
-					console.log("Waiting 30 secongs")
+					console.log("Waiting 10 secongs")
 					worker.syncAttempts++;
 					setTimeout(function(){
 						worker.sync(maxAttempts, callback, errorCallback);
-					}, 30000);
+					}, 15000);
 				}
 			} else {
 				
@@ -220,7 +220,7 @@ var Worker = (function(){
 				worker.isBuildingRunners = false;
 				worker.isSyncing = false;
 				worker.syncAttempts = 0;
-				callback(worker);
+				callback(null, worker);
 			}
 		});
 	};
@@ -230,7 +230,7 @@ var Worker = (function(){
 		// Once active the droplet begins to create runners
 		var maxMemoryUsage = args.maxMemoryUsage || config.maxMemoryUsage || 80;
 		var worker_uuid = utils.uuid();
-		// var phone_home = config.home || "/worker/ping";
+		var phone_home = config.home || "/worker/ping";
 
 		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
 
@@ -271,11 +271,11 @@ var Worker = (function(){
 		
 		createScript = `echo "${script}" | cat > /home/virt/allocate_runners.sh`;
 		
-		makeScriptExecutable = `chmod virt +x /home/virt/allocate_runners.sh`;
+		makeScriptExecutable = `chmod o+x /home/virt/allocate_runners.sh`;
 		
 		setupCrontab = `echo "*/${interval} * * * * /home/virt/allocate_runners.sh > /home/virt/allocate_runners.log 2>&1" | crontab -u virt -`;
 		
-		return `${createScript} && ${makeScriptExecutable} && ${setupCrontab};`;
+		return `#!/bin/bash\n\n${createScript} && ${makeScriptExecutable} && ${setupCrontab};`;
 	};
 
 	proto.startRunners = function(args, count){
@@ -385,7 +385,7 @@ var WorkerCollection = (function(){
 		var count = 0;
 		config = config || workers.settings;
 		Worker.initialize({
-			"callback": function(worker){
+			"callback": function(error, worker){
 				console.log("Seeded runners on", worker.name);
 				workers.push(worker);
 				worker.register();
@@ -393,6 +393,7 @@ var WorkerCollection = (function(){
 			},
 			"errorCallback": function(error, worker){
 				// destroy worker
+				workers.currentCreating--;
 			}
 		}, config);
 	};
@@ -450,37 +451,46 @@ var WorkerCollection = (function(){
 		});
 	};
 
-	workers.checkForZombies = function(){
+	workers.checkForZombies = function(callback){
 		// check to make sure all works are used or usable.
-		
-		let zombies = 0;
+		if (workers.length === 0) callback();
+		let 
+			zombies = 0, 
+			syncedCount = workers.length,
+			workerCleanUp = function(error, worker){
+				console.log(`Zombie! Worker ${worker.name}, destroying.`);
+				workers.destroy(worker);
+				zombies++;
+				if(!--count) callback();
+			};
+
 
 		for(let worker of workers){
 			console.log(`Checking if ${worker.name} is a zombie worker.`);
 			// if a runner has no available runners and no used runners, its a
 			// zombie. This should happen when a newer image ID has been added
 			// and old workers slowly lose there usefulness.
-
-			if(worker.isZombie()){
-				console.log(`Zombie! Worker ${worker.name}, destroying.`);
-				workers.destroy(worker);
-				zombies++;
-			}
+			worker.sync(function(error, worker){
+				if(worker.isZombie()) workerCleanUp(error, worker);
+			}, workerCleanUp);
 		}
-
-		return zombies;
 	};
 
 	workers.checkBalance = function(){
 		console.log(`${(new Date())} Checking balance.`);
 
-		workers.checkForZombies();
+		workers.checkForZombies(function(){
+			// if there are workers being created, stop scale up and down check
+			var skipBalance = workers.currentCreating + workers.length >= workers.settings.min;
+			if(workers.currentCreating && skipBalance){
+				return console.log(`Killing balance, workers are being created.`);
+			}
 
-		// if there are workers being created, stop scale up and down check
-		var skipBalance = workers.currentCreating + workers.length >= workers.settings.min
-		if(workers.currentCreating && skipBalance){
-			return console.log(`Killing balance, workers are being created.`);
-		}
+			workers.balance();
+		});
+	};
+
+	workers.balance = function(){
 
 		// count workers and locate oldest worker
 		var oldestWorker, isNotOlder, workerCount = 0;
@@ -533,6 +543,7 @@ var WorkerCollection = (function(){
 			`);
 		}
 
+
 	};
 
 	workers.start = function(interval){
-- 
2.34.1


From 3dcfbf4d771862e4b9a363c382d298bc29186df4 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 24 Oct 2017 16:42:17 -0400
Subject: [PATCH 41/42] patch for Worker.sync

---
 routes/worker_collection.js | 83 +++++++++++--------------------------
 1 file changed, 24 insertions(+), 59 deletions(-)

diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index 5890bef..d93e703 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -94,6 +94,13 @@ var Worker = (function(){
 		});
 
 		worker.availrunners = [];
+		// need map of used runners
+		// need list of all runners
+		// - sync should probably populate the all runners container
+		// - availrunners should the diff of used runners and all runners
+		
+		// runners should probably indicate when they have been used.
+
 		worker.ip = worker.publicIP;
 		worker.usedrunners = 0;
 		worker.age = +(new Date());
@@ -111,8 +118,7 @@ var Worker = (function(){
 		// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
 		var runner = this.availrunners.pop();
 		this.usedrunners++;
-		runner.setTimeout();
-					
+		runner.setTimeout();	
 		return runner;
 	};
 
@@ -151,9 +157,9 @@ var Worker = (function(){
 
 	// When should this be called
 	proto.sync = function(callback, errorCallback, maxAttempts){
-		maxAttempts = maxAttempts || maxSyncAttempts;
-
 		var worker = this;
+		
+		maxAttempts = maxAttempts || maxSyncAttempts;
 		worker.isSyncing = true;
 		callback = callback || __empty;
 		errorCallback = errorCallback || __empty;
@@ -164,9 +170,6 @@ var Worker = (function(){
 		// - check memory and check runners
 		// - when does start runners get called?
 
-		// worker.ramPercentUsed();
-
-
 		lxc.exec('lxc-ls --fancy', worker.ip, function(data, error, stderr){
 			if (error){
 				console.log("Sync Error: \n", error);
@@ -175,7 +178,7 @@ var Worker = (function(){
 						errorCallback(error, worker);
 					}, 0);
 				} else {
-					console.log("Waiting 10 secongs")
+					console.log("Waiting 15 seconds")
 					worker.syncAttempts++;
 					setTimeout(function(){
 						worker.sync(maxAttempts, callback, errorCallback);
@@ -203,15 +206,18 @@ var Worker = (function(){
 				}
 				console.log(`RUNNERS FOUND[=> ${worker.ip}`);
 				console.log(`RUNNERS FOUND[=>`, runners);
+				// bad
 				worker.availrunners = [];
 
 				for (let idx = 0, stop = runners.length; idx < stop; idx++){
-					if(runners[idx].state !== "STOPPED"){
+					if(runners[idx].state !== "STOPPED" && !Runner.get(worker.name + ':' + runners[idx].name)){
 						var runner = Runner.create({
 							"name": runners[idx].name,
+							"ipv4": runners[idx].ipv4,
 							"worker": worker,
 							"label": worker.name + ':' + runners[idx].name
 						});
+
 						worker.availrunners.push(runner);
 					}
 				}
@@ -225,13 +231,17 @@ var Worker = (function(){
 		});
 	};
 
-	proto.initialize = function(args, config){
+	proto.initialize = function(params, config){
 		// Create droplet
 		// Once active the droplet begins to create runners
 		var maxMemoryUsage = args.maxMemoryUsage || config.maxMemoryUsage || 80;
 		var worker_uuid = utils.uuid();
 		var phone_home = config.home || "/worker/ping";
 
+
+		var callback = params.callback || __empty;
+		var errorCallback = params.errorCallback || empty;
+
 		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
 
 			doapi.dropletToActive({
@@ -250,11 +260,10 @@ var Worker = (function(){
 					data.worker_uuid = worker_uuid;
 					var worker = Worker.create(data);
 					
-					// Just send the job over and set a timeout 
-					// to wait before checking runners
+					// wait for boot before syncing runners
 					setTimeout(function(){
-						worker.sync(args.callback);
-					}, 90000);
+						worker.sync(callback, errorCallback);
+					}, 75000);
 				}
 			});
 		});
@@ -278,50 +287,6 @@ var Worker = (function(){
 		return `#!/bin/bash\n\n${createScript} && ${makeScriptExecutable} && ${setupCrontab};`;
 	};
 
-	proto.startRunners = function(args, count){
-		if ( count > 3) return;
-		// onStart is not necessary
-		var worker = this;
-		args.stopPercent = args.stopPercent || 80;
-		args.callback = args.callback || __empty;
-		args.errorCallback = args.errorCallback || __empty;
-
-		// dont make runners on out dated workers
-		if(!worker || worker.settings.image > worker.image.id || worker.isBuildingRunners){
-			if(worker) {
-				console.log(`
-					Blocked worker(${worker.image.id}), current image ${worker.settings.image}.
-					Building: ${worker.isBuildingRunners}
-				`);
-			}
-			return;
-		}
-
-		worker.isBuildingRunners = true;
-		fs.readFile(__dirname + "/../allocate_runners.sh", function(error, file){
-			var command = worker.__buildCommand(file, args.stopPercent);
-				
-			lxc.exec(command, worker.ip, function(data, error, stderr){
-				if (error) { 
-					console.log("ERROR: ",stderr);
-					setTimeout(function(){
-						count = count || 0;
-						worker.startRunners(args, ++count);
-					}, 0);
-				} else {
-					console.log("SUCCESS: ", data , worker.ip);
-				}
-				// Just send the job over and set a timeout 
-				// to wait before checking runners
-				setTimeout(function(){
-					worker.sync(args.callback);
-					// 900000 < thats 15 min not 90 sec
-				}, 90000);
-			});
-
-		});
-	};
-
 	return proto;
 })();
 
@@ -491,7 +456,7 @@ var WorkerCollection = (function(){
 	};
 
 	workers.balance = function(){
-
+		console.log(`BALANCING: ${(new Date())}`);
 		// count workers and locate oldest worker
 		var oldestWorker, isNotOlder, workerCount = 0;
 
-- 
2.34.1


From fd919d0b91cbeddfcbc1c5db7be60aa7c3f23b89 Mon Sep 17 00:00:00 2001
From: Thomas Harvey <thomasharvey24@gmail.com>
Date: Tue, 24 Oct 2017 23:08:10 -0400
Subject: [PATCH 42/42] another set of edits that reveal more regressions

---
 routes/api.js               | 71 +++++++---------------------
 routes/worker_collection.js | 94 +++++++++++++++++++++++++++++++------
 2 files changed, 96 insertions(+), 69 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index 2cc177c..45f6736 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -2,8 +2,8 @@
 
 var express = require('express');
 var router = express.Router();
+// what is util for??
 var util = require('util');
-var request = require('request');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
 
@@ -13,58 +13,6 @@ console.log('========STARTING===========');
 
 workers.start();
 
-var attemptRun = function(req, res, runner, count){
-	count = count || 0;
-	console.log(`Runner starting attempt ${count}.`);
-
-	if(!runner){
-		console.log(`No runner available!`);
-		res.status(503);
-		return res.json({error: 'No runners, try again soon.'});
-	}
-
-	// TODO: Configurable
-	if(count > 2){
-		console.log(`Runner attempt failed, to many requests!`);
-		return res.status(400).json({error: 'Runner restarted to many times'});
-	}
-
-	var httpOptions = {
-		url: 'http://' + runner.worker.ip,
-		headers: {
-			Host: runner.name
-		},
-		body: JSON.stringify({
-			code: req.body.code
-		})
-	};
-
-	return request.post(httpOptions, function(error, response, body){
-		// console.log('runner response:', arguments)
-		if(error || response.statusCode !== 200) {
-			return attemptRun(req, res, workers.getAvailableRunner(), ++count);
-		}
-		
-		body = JSON.parse(body);
-
-		if(req.query.once){
-			res.json(body);
-			// 0 here does nothing
-			// return runnerFree(runner, 0);
-			return runner.free();
-		}
-
-		workers.setRunner(runner);
-		body['ip'] = runner.label;
-		body['rname'] = runner.name;
-		body['wname'] = runner.worker.name;
-		res.json(body);
-
-		// runnerTimeout(runner);
-		runner.setTimeout();
-	});
-};
-
 // Why is this a GET?
 router.get('/stop/:name', function(req, res, next){
 	return lxc.stop(req.params.name, function(data){
@@ -105,8 +53,21 @@ router.get('/ping/:runner', function(req, res, next){
 
 router.post('/run/:runner?', function (req, res, next){
 	console.log(`Request runner route!`);
-	var runner = workers.getAvailableRunner(workers.getRunner(req.params.runner));
-	return attemptRun(req, res, runner);
+	
+	return workers.attemptRun(
+		req.body.code, req.query.once, req.params.runner, 
+		(body) => {
+			res.json(body);
+		}, 
+		(error, statusCode) => {
+			if (statusCode === 503){
+				res.status(503);
+				return res.json({error: 'No runners, try again soon.'});
+			} else if (statusCode === 400){
+				return res.status(400).json({error: 'Runner restarted too many times'});
+			}
+		}
+	);
 });
 
 module.exports = router;
diff --git a/routes/worker_collection.js b/routes/worker_collection.js
index d93e703..95f6cd1 100644
--- a/routes/worker_collection.js
+++ b/routes/worker_collection.js
@@ -1,6 +1,7 @@
 'use strict';
 
 var jsonfile = require('jsonfile');
+var request = require('request');
 var lxc = require('../lxc');
 var doapi = require('../doapi')();
 var settings = require('./workers.json');
@@ -38,7 +39,7 @@ var Runner = (function(){
 	proto.get = function(label){
 		return proto.runnerMap[label];
 	};
-	
+
 
 	proto.create = function(config){
 		var runner = Object.create(proto);
@@ -47,9 +48,13 @@ var Runner = (function(){
 	};
 
 	proto.free = function(){
+		// track how often this happens in a minute
 		var runner = this;
+
 		lxc.stop(runner.name, runner.worker.ip);
+		// this should be done by the worker
 		runner.worker.usedrunners--;
+
 		if(runner.hasOwnProperty('timeout')){
 			clearTimeout(runner.timeout);
 		}
@@ -57,8 +62,11 @@ var Runner = (function(){
 		if(runner.hasOwnProperty('cleanUp')){
 			runner.cleanUp();
 		}
-		// TODO: Determine if this call is even needed
-		// runner.worker.sync();
+		// worker has more space now
+		// worker should track this
+		setTimeout(()=>{
+			runner.worker.populate();
+		}, 0);
 	};
 
 	proto.setTimeout = function(time){
@@ -94,13 +102,6 @@ var Worker = (function(){
 		});
 
 		worker.availrunners = [];
-		// need map of used runners
-		// need list of all runners
-		// - sync should probably populate the all runners container
-		// - availrunners should the diff of used runners and all runners
-		
-		// runners should probably indicate when they have been used.
-
 		worker.ip = worker.publicIP;
 		worker.usedrunners = 0;
 		worker.age = +(new Date());
@@ -113,6 +114,14 @@ var Worker = (function(){
 		return worker;
 	};
 
+	proto.populate = function(callback){
+		callback = callback || __empty;
+		return lxc.startEphemeral(
+			"crunner-batch-$RANDOM-id-$RANDOM", "crunner0", this.ip, callback
+		);
+	};
+
+
 	proto.getRunner = function(){
 		if(this.availrunners.length === 0) return false;
 		// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
@@ -168,7 +177,6 @@ var Worker = (function(){
 		// mainly to update the active runners on the worker
 		// potentially collect stats about the droplet as well
 		// - check memory and check runners
-		// - when does start runners get called?
 
 		lxc.exec('lxc-ls --fancy', worker.ip, function(data, error, stderr){
 			if (error){
@@ -206,11 +214,13 @@ var Worker = (function(){
 				}
 				console.log(`RUNNERS FOUND[=> ${worker.ip}`);
 				console.log(`RUNNERS FOUND[=>`, runners);
-				// bad
 				worker.availrunners = [];
 
 				for (let idx = 0, stop = runners.length; idx < stop; idx++){
-					if(runners[idx].state !== "STOPPED" && !Runner.get(worker.name + ':' + runners[idx].name)){
+					if(runners[idx].state === "STOPPED" || Runner.get(worker.name + ':' + runners[idx].name)){
+						continue;
+					} else {
+
 						var runner = Runner.create({
 							"name": runners[idx].name,
 							"ipv4": runners[idx].ipv4,
@@ -234,7 +244,7 @@ var Worker = (function(){
 	proto.initialize = function(params, config){
 		// Create droplet
 		// Once active the droplet begins to create runners
-		var maxMemoryUsage = args.maxMemoryUsage || config.maxMemoryUsage || 80;
+		var maxMemoryUsage = params.maxMemoryUsage || config.maxMemoryUsage || 80;
 		var worker_uuid = utils.uuid();
 		var phone_home = config.home || "/worker/ping";
 
@@ -524,6 +534,62 @@ var WorkerCollection = (function(){
 		});
 	};
 
+	workers.attemptRun = function(code, once, runnerLabel, callback, errorCallback, count){
+		// PARAMS: code, once, runnerLabel, callback, errorCallback, count;
+		
+		var runner = workers.getAvailableRunner(
+			workers.getRunner(runnerLabel)
+		);
+
+		if(!runner){
+			console.log(`No runner available!`);
+			return errorCallback(null, 503);
+		}
+
+		count = count || 0;
+
+		console.log(`Runner starting attempt ${count}.`);
+
+
+		if(count > 2){
+			console.log(`Runner attempt failed, to many requests!`);
+			return errorCallback(null, 400);
+		}
+
+		var httpOptions = {
+			url: 'http://' + runner.worker.ip,
+			headers: {
+				Host: runner.name
+			},
+			body: JSON.stringify({
+				code: code
+			})
+		};
+
+		// Move the http stuff to the runner
+		return request.post(httpOptions, function(error, response, body){
+			// console.log('runner response:', arguments)
+			
+			if(error || response.statusCode !== 200) {
+				return workers.attemptRun(code, once, void 0, callback, errorCallback, ++count);
+			}
+			
+			body = JSON.parse(body);
+			
+			if(once){
+				runner.free();
+			} else {
+				runner.setTimeout();
+				body['ip'] = runner.label;
+				body['rname'] = runner.name;
+				body['wname'] = runner.worker.name;
+				workers.setRunner(runner);
+			}
+			return callback(body);
+		});
+	};
+
+
 	workers.add = function(newWorkers){
 		newWorkers.forEach(function(worker){
 			workers.push(worker);
-- 
2.34.1