updating file name
This commit is contained in:
Thomas Harvey
382
routes/worker_collection.js
Normal file
382
routes/worker_collection.js
Normal file
@@ -0,0 +1,382 @@
|
||||
'use strict';
|
||||
|
||||
var jsonfile = require('jsonfile');
|
||||
var lxc = require('../lxc');
|
||||
var doapi = require('../doapi')();
|
||||
var settings = require('./workers.json');
|
||||
|
||||
|
||||
var Runner = (function(){
|
||||
var proto = {};
|
||||
|
||||
proto.create = function(config){
|
||||
var runner = Object.create(proto);
|
||||
var __empty = function(){};
|
||||
Object.assign(runner, config);
|
||||
runner.afterFreed = runner.afterFreed || __empty;
|
||||
return runner;
|
||||
};
|
||||
|
||||
proto.free = function(callback){
|
||||
var runner = this;
|
||||
lxc.stop(runner.name, runner.worker.ip);
|
||||
runner.worker.usedrunners--;
|
||||
if(runner.hasOwnProperty('timeout')){
|
||||
clearTimeout(runner.timeout);
|
||||
};
|
||||
runner.afterFreed();
|
||||
};
|
||||
|
||||
proto.setTimeout = function(time){
|
||||
time = time || 60000; // 1 minutes
|
||||
var runner = this;
|
||||
if(runner.hasOwnProperty('timeout')){
|
||||
clearTimeout(runner.timeout);
|
||||
}
|
||||
|
||||
return runner.timeout = setTimeout(function(){
|
||||
runner.free();
|
||||
}, time);
|
||||
};
|
||||
|
||||
})();
|
||||
|
||||
|
||||
var Worker = (function(){
|
||||
var proto = {};
|
||||
|
||||
proto.create = function(config){
|
||||
var worker = Object.create(proto);
|
||||
|
||||
worker.networks.v4.forEach(function(value){
|
||||
worker[value.type+'IP'] = value.ip_address;
|
||||
});
|
||||
|
||||
worker.availrunners = [];
|
||||
worker.ip = worker.publicIP;
|
||||
worker.usedrunners = 0;
|
||||
worker.index = workers.length;
|
||||
|
||||
return worker;
|
||||
};
|
||||
|
||||
proto.getRunner = function(){
|
||||
if(this.availrunners.length === 0) return false;
|
||||
// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
|
||||
var runner = this.availrunners.pop();
|
||||
this.usedrunners++;
|
||||
runner.setTimeout();
|
||||
|
||||
return runner;
|
||||
};
|
||||
|
||||
|
||||
proto.ramPercentUsed = function(callback){
|
||||
// checks the percent of ram used on a worker.
|
||||
return lxc.exec(
|
||||
"python3 -c \"a=`head /proc/meminfo|grep MemAvail|grep -Po '\\d+'`;t=`head /proc/meminfo|grep MemTotal|grep -Po '\\d+'`;print(round(((t-a)/t)*100, 2))\"",
|
||||
this.ip,
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
proto.destroy = function(){
|
||||
var worker = this;
|
||||
return doapi.dropletDestroy(this.id, function(body) {
|
||||
console.log('Deleted worker', this.name);
|
||||
});
|
||||
};
|
||||
})();
|
||||
|
||||
|
||||
var WorkerCollection = (function(){
|
||||
// works array constructor. This will hold the works(order by creation) and all
|
||||
// the methods interacting with the workers.
|
||||
|
||||
// base array that will be the workers objects.
|
||||
var workers = [];
|
||||
|
||||
var tagPrefix = settings.tagPrefix || 'clwV';
|
||||
|
||||
workers.runnerMap = {};
|
||||
|
||||
// persistent settings object
|
||||
// .image is the currently used Digital Ocean snap shot ID
|
||||
// .lastSnapShotId is the previous ID used Digital Ocean snap shot
|
||||
// .version is the current worker version
|
||||
// .size is the base Droplet size for worker creation
|
||||
// .min is the minimum amount of workers that should exist
|
||||
// .max is the maximum amount of works that ca exist
|
||||
// .minAvail is the amount of empty workers there should be
|
||||
workers.settings = settings;
|
||||
|
||||
// How many droplets are currently in the process of being created. It takes
|
||||
// about 3 minutes to create a worker.
|
||||
workers.currentCreating = 0;
|
||||
|
||||
|
||||
|
||||
workers.setRunner = function(runner){
|
||||
runnerMap[runner.label] = runner;
|
||||
};
|
||||
|
||||
|
||||
workers.getRunner = function(label){
|
||||
return runnerMap[runner.label];
|
||||
};
|
||||
|
||||
|
||||
workers.getAvailableRunner = function(runner){
|
||||
for(let worker of workers){
|
||||
if(worker.availrunners.length === 0) continue;
|
||||
if(runner && runner.worker.index <= worker.index) break;
|
||||
// if(runner) runnerFree(runner);
|
||||
if(runner) runner.free();
|
||||
|
||||
return worker.getRunner();
|
||||
}
|
||||
|
||||
if(runner) return runner;
|
||||
};
|
||||
|
||||
workers.create = function(config){
|
||||
// manages the creation of a work from first call to all runners seeded
|
||||
|
||||
// dont create more workers then the settings file allows
|
||||
if(workers.currentCreating > workers.settings.max ) return false;
|
||||
workers.currentCreating++;
|
||||
|
||||
config = config || workers.settings;
|
||||
|
||||
doapi.dropletToActive({
|
||||
name: 'clw' + config.version + '-' + (Math.random()*100).toString().slice(-4),
|
||||
image: config.image,
|
||||
size: config.size,
|
||||
onCreate: function(data){
|
||||
doapi.dropletSetTag(
|
||||
tagPrefix + config.version,
|
||||
data.droplet.id
|
||||
);
|
||||
},
|
||||
onActive: function(data, args){
|
||||
workers.startRunners({
|
||||
worker: Worker.create(data),
|
||||
onStart: function(runner, args){
|
||||
workers.push(args.worker);
|
||||
doapi.domianAddRecord({
|
||||
domain: "codeland.us",
|
||||
type: "A",
|
||||
name: "*." + worker.name + ".workers",
|
||||
data: worker.publicIP
|
||||
});
|
||||
args.onStart = function(){};
|
||||
},
|
||||
onDone: function(args){
|
||||
console.log("Seeded runners on", worker.name);
|
||||
workers.currentCreating--;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
workers.__workersId = function(argument){
|
||||
// create array of all current worker Digital Ocean ID
|
||||
return workers.map(function(item){
|
||||
return item.id;
|
||||
});
|
||||
};
|
||||
|
||||
workers.destroy = function(worker){
|
||||
// removes last one
|
||||
// todo: If worker is passed, check for it in the workers array and
|
||||
// remove it if found.
|
||||
if ( worker ){
|
||||
var worker_idx = workers.indexOf(worker);
|
||||
if (~worker_idx){
|
||||
workers.splice(worker_idx, 1);
|
||||
return worker.destroy();
|
||||
}
|
||||
} else {
|
||||
worker = workers.pop();
|
||||
return worker.destroy();
|
||||
}
|
||||
};
|
||||
|
||||
workers.destroyByTag = function(tag){
|
||||
// Delete works that with
|
||||
|
||||
tag = tag || tagPrefix + workers.settings.version;
|
||||
let currentIDs = workers.__workersId();
|
||||
|
||||
let deleteDroplets = function(droplets){
|
||||
if(droplets.length === 0) return true;
|
||||
let droplet = droplets.pop();
|
||||
if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
|
||||
|
||||
doapi.dropletDestroy(droplet.id, function(body){
|
||||
setTimeout(deleteDroplets, 1000, droplets);
|
||||
if(!droplets.length) console.log(`Finished deleting workers tagged ${tag}.`);
|
||||
});
|
||||
}
|
||||
|
||||
doapi.dropletsByTag(tag, function(data){
|
||||
data = JSON.parse(data);
|
||||
console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
|
||||
data['droplets'].map(function(item){
|
||||
return item.name+' | '+item.id;
|
||||
})
|
||||
);
|
||||
|
||||
deleteDroplets(data['droplets']);
|
||||
});
|
||||
};
|
||||
|
||||
workers.startRunners = function(args){
|
||||
// console.log('starting runners on', args.worker.name, args.worker.ip)
|
||||
|
||||
// dont make runners on out dated workers
|
||||
if(!args.worker || workers.settings.image > args.worker.image.id){
|
||||
console.log(`Blocked outdated worker(${args.worker.image.id}), current image ${workers.settings.image}.`)
|
||||
return ;
|
||||
}
|
||||
|
||||
// percent of used RAM to stop runner creation
|
||||
args.stopPercent = args.stopPercent || 80;
|
||||
args.onStart = args.onStart || function(){};
|
||||
args.onDone = args.onDone || function(){};
|
||||
|
||||
args.worker.ramPercentUsed(function(usedMemPercent){
|
||||
if(usedMemPercent > args.stopPercent ){
|
||||
console.log('using', String(usedMemPercent).trim(),
|
||||
'percent memory, stopping runner creation!', args.worker.availrunners.length,
|
||||
'created on ', args.worker.name
|
||||
);
|
||||
args.onDone(args);
|
||||
return ;
|
||||
}
|
||||
|
||||
var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
|
||||
// console.log('Free ram check passed!')
|
||||
lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
|
||||
if(!data.ip){
|
||||
return setTimeout(workers.startRunners, 0, args);
|
||||
}
|
||||
// console.log('started runner on', args.worker.name)
|
||||
|
||||
var runner = Runner.create({
|
||||
ip: data.ip,
|
||||
name: name,
|
||||
worker: args.worker,
|
||||
label: args.worker.name + ':' + name,
|
||||
afterFreed: function(runner){
|
||||
delete workers.runnerMap[runner.label];
|
||||
console.log(`Runner freed ${runner.label}.`, runner.worker);
|
||||
// Why does this need to run here?
|
||||
workers.startRunners({worker: runner.worker});
|
||||
}
|
||||
});
|
||||
|
||||
args.onStart(runner, args);
|
||||
|
||||
args.worker.availrunners.push(runner);
|
||||
|
||||
setTimeout(workers.startRunners, 0, args);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
workers.checkForZombies = function(){
|
||||
// check to make sure all works are used or usable.
|
||||
|
||||
let zombies = 0;
|
||||
|
||||
for(let worker of workers){
|
||||
console.log(`Checking if ${worker.name} is a zombie worker.`);
|
||||
// if a runner has no available runners and no used runners, its a
|
||||
// zombie. This should happen when a newer image ID has been added
|
||||
// and old workers slowly lose there usefulness.
|
||||
if(worker.availrunners.length === 0 && worker.usedrunners === 0){
|
||||
workers.splice(workers.indexOf(worker), 1);
|
||||
console.log(`Zombie! Worker ${worker.name}, destroying.`);
|
||||
workers.destroy(worker);
|
||||
zombies++;
|
||||
}
|
||||
}
|
||||
|
||||
return zombies;
|
||||
};
|
||||
|
||||
workers.checkBalance = function(){
|
||||
console.log(`Checking balance.`);
|
||||
|
||||
workers.checkForZombies();
|
||||
|
||||
// if there are workers being created, stop scale up and down check
|
||||
if(workers.currentCreating+workers.length < workers.settings.min) {
|
||||
null;
|
||||
} else if(workers.currentCreating){
|
||||
return console.log(`Killing balance, workers are being created.`);
|
||||
}
|
||||
|
||||
// hold amount of workers with no used runners
|
||||
var lastMinAval = 0;
|
||||
|
||||
// check to make sure the `workers.settings.minAvail` have free runners
|
||||
for(let worker of workers.slice(-workers.settings.minAvail)){
|
||||
// INVERT this conditional
|
||||
if(worker.usedrunners !== 0){
|
||||
lastMinAval++;
|
||||
}else{
|
||||
// no need to keep counting, workers need to be created
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(lastMinAval > workers.settings.minAvail){
|
||||
// Remove workers if there are more then the settings states
|
||||
console.log(
|
||||
`Last ${workers.settings.minAvail} workers not used, killing last worker`,
|
||||
'lastMinAval:', lastMinAval,
|
||||
'minAvail:', workers.settings.minAvail,
|
||||
'workers:', workers.length
|
||||
);
|
||||
|
||||
return workers.destroy();
|
||||
|
||||
} else if(lastMinAval < workers.settings.minAvail){
|
||||
// creates workers if the settings file demands it
|
||||
console.log(
|
||||
'last 3 workers have no free runners, starting worker',
|
||||
'lastMinAval:', lastMinAval,
|
||||
'minAvail:', workers.settings.minAvail,
|
||||
'workers:', workers.length
|
||||
);
|
||||
|
||||
return workers.create();
|
||||
}
|
||||
|
||||
};
|
||||
workers.settingsSave = function(){
|
||||
// save the live settings file to disk
|
||||
|
||||
jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) {
|
||||
console.error(err);
|
||||
});
|
||||
};
|
||||
|
||||
workers.add = function(newWorkers){
|
||||
newWorkers.forEach(function(worker){
|
||||
workers.push(worker);
|
||||
});
|
||||
};
|
||||
// does this have to be last?
|
||||
// make sure Digital Ocean has a tag for the current worker version
|
||||
doapi.tagCreate(tagPrefix + workers.settings.version);
|
||||
|
||||
return workers;
|
||||
|
||||
})();
|
||||
|
||||
|
||||
module.exports = WorkerCollection;
|
||||
Reference in New Issue
Block a user