wmantly/lxc_manager_node
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #21 from wmantly/omg

Browse Source 
Omg
This commit is contained in:
William Mantly 2017-09-10 00:17:47 -04:00 committed by GitHub
commit 70091ddbdb
8 changed files with 319 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app.js
View File

@ -18,6 +18,12 @@ app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use(function(req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
next();
});
app.use('/', require('./routes/index'));
app.use('/api', require('./routes/api'));

47
doapi.js
View File

@ -34,6 +34,7 @@ api = function(key){
};
this.dropletSetTag = function(tag, dropletID, callback) {
callback = callback || function(){};
var data = {
resources: [
{
@ -55,6 +56,8 @@ api = function(key){
};
this.dropletCreate = function(args, callback){
callback = callback || function(){};
var data = {
name: args.name, // || return false,
region: args.region || 'nyc3',
@ -77,7 +80,35 @@ api = function(key){
});
}
this.dropletToActive = function(args){
args.__doapi = this; // hold the DO api in the agrs scope
args.onCreated = args.onCreate || function(){};
this.dropletCreate(args, function(data){
data = JSON.parse(data);
args.onCreate(data, args);
// check if the server is ready, giving time to allow
// digital ocean to do its thing
setTimeout(function check(id, args){
time = args.time || 10000;
args.__doapi.dropletInfo(id, function (data){
var droplet = JSON.parse(data)['droplet'];
if(droplet.status == 'active'){
return args.onActive(droplet, args);
}else{
setTimeout(function(check, id){
check(id, args);
}, time, check, droplet.id);
}
});
}, 70000, data.droplet.id, args);
});
};
this.dropletDestroy = function(dropletID, callback){
callback = callback || function(){};
var options = {
url: this.BASEURL+'droplets/'+dropletID,
headers: this.headers
@ -101,6 +132,20 @@ api = function(key){
});
};
this.tagCreate = function(tag, callback){
callback = callback || function(){};
var options = {
url: this.BASEURL+'tags',
headers: this.headers,
body: JSON.stringify({name: tag})
};
this.calls++;
return request.post(options, function(error, response, body){
return callback(body, response, error);
});
};
this.tagsList = function(callback){
var options = {
url: this.BASEURL+'tags',
@ -114,7 +159,7 @@ api = function(key){
};
this.domianAddRecord = function(args, callback){
callback = callback || function(){};
var options = {
url: this.BASEURL+'domains/'+ args.domain +'/records',
headers: this.headers,

3
lxc.js
View File

@ -1,10 +1,11 @@
'use strict';
var exec = require('child_process').exec;
function sysExec(command, ip, callback){
ip = ip || '104.236.77.157';
command = new Buffer(command).toString('base64')
command = 'ssh -o StrictHostKeyChecking=no virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"';
command = 'ssh -i ~/.ssh/clw_rsa -o StrictHostKeyChecking=no virt@'+ ip + ' "echo ' + command + '|base64 --decode|bash"';
// command = 'unset XDG_SESSION_ID XDG_RUNTIME_DIR; cgm movepid all virt $$; ' + command;
return exec(command, (function(callback){

5
package.json
View File

@ -11,10 +11,11 @@
"debug": "~2.2.0",
"ejs": "~2.3.3",
"express": "~4.13.1",
"jsonfile": "^2.3.1",
"morgan": "~1.6.1",
"serve-favicon": "~2.3.0",
"node.extend": "~1.1.5",
"redis": "~2.4.2",
"request": "~2.69.0"
"request": "~2.69.0",
"serve-favicon": "~2.3.0"
}
}

461
routes/api.js
View File

@ -4,88 +4,69 @@ var express = require('express');
var router = express.Router();
var util = require('util');
var request = require('request');
var jsonfile = require('jsonfile');
var lxc = require('../lxc');
var doapi = require('../doapi')();
var workerSnapID = 'V1'
// console.log = function(){};
var label2runner = {};
var isCheckingWorkers = false;
var workers = (function(){
var workers = [];
workers.currentCreatingMax = 2;
workers.settings = require('./workers.json');
workers.currentCreating = 0;
workers.checkDroplet = function(id, time){
time = time || 10000;
return doapi.dropletInfo(id, function(data){
var worker = JSON.parse(data)['droplet'];
if(worker.status == 'active'){
// console.log('Droplet is now active, starting runners in 20 seconds');
return setTimeout(function(worker){
// console.log('Ready to start runners!');
workers.startRunners(workers.makeWorkerObj(worker), true);
workers.currentCreating--;
}, 20000, worker);
}else{
// console.log('Worker not ready, check again in ', time, 'MS');
return setTimeout(function(){
workers.checkDroplet(id);
}, time);
}
});
};
workers.create = function(){
if(workers.currentCreating > workers.currentCreatingMax ) return false;
return doapi.dropletCreate({
name: 'clw'+workerSnapID+'-'+(Math.random()*100).toString().slice(-4),
image: '18473675',
size: '2gb'
}, function(data){
data = JSON.parse(data);
workers.currentCreating++;
setTimeout(function(dopletNewID){
return workers.checkDroplet(dopletNewID);
}, 70000, data.droplet.id);
return doapi.dropletSetTag('clworker', data.droplet.id, function(){});
if(workers.currentCreating > workers.settings.max ) return false;
workers.currentCreating++;
doapi.dropletToActive({
name: 'clw'+workers.settings.version+'-'+(Math.random()*100).toString().slice(-4),
image: workers.settings.image,
size: workers.settings.size,
onCreate: function(data){
console.log(data)
doapi.dropletSetTag('clwV'+workers.settings.version, data.droplet.id);
},
onActive: function(worker, args){
workers.startRunners({
worker: workers.makeWorkerObj(worker),
onStart: function(runner, args){
workers.push(args.worker);
doapi.domianAddRecord({
domain: "codeland.us",
type: "A",
name: "*."+worker.name+".workers",
data: worker.publicIP
});
args.onStart = function(){};
},
onDone: function(args){
console.log("done with workers");
}
});
workers.currentCreating--;
}
});
};
workers.destroy = function(worker){
var worker = worker || workers.pop();
return doapi.dropletDestroy(worker.id, function(body) {
console.log('body of destroy', body)
});
};
workers.makeWorkerObj = function(worker){
worker.networks.v4.forEach(function(value){
worker[value.type+'IP'] = value.ip_address;
});
worker.availrunners = [];
worker.ip = worker.privateIP;
worker.ip = worker.publicIP;
worker.usedrunner = 0;
worker.index = workers.length,
worker.index = workers.length;
worker.getRunner = function(){
if(this.availrunners.length === 0) return false;
// console.log('getting runner from ', worker.name, ' avail length ', this.availrunners.length);
var runner = this.availrunners.pop();
this.usedrunner++;
lxcTimeout(runner);
label2runner[runner.label] = runner;
runnerTimeout(runner);
return runner;
}
@ -99,90 +80,127 @@ var workers = (function(){
};
workers.destroyOld = function(){
var currentIDs = workers.__workersId()
doapi.dropletsByTag('clworker', function(data){
data = JSON.parse(data);
console.log('current worker ids', currentIDs)
console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){
return item.name+'| '+item.id;
}));
data['droplets'].forEach(function(worker){
console.log('checking', worker.name, worker.id)
if(~currentIDs.indexOf(worker.id)) return false;
workers.destroy = function(worker){
var worker = worker || workers.pop();
return doapi.dropletDestroy(worker.id, function(body) {
console.log('body of destroy', body);
});
};
console.log('found old droplet, killing it');
setTimeout(function(id) {
// body...
doapi.dropletDestroy(id, function(body){console.log('delete body', body)});
}, Math.floor(Math.random() * 9) + 1, worker.id)
workers.destroyOld = function(tag){
tag = tag || 'clwV'+workers.settings.version;
var currentIDs = workers.__workersId();
var deleteDroplets = function(droplets){
if(droplets.length === 0) return true;
var droplet = droplets.pop();
if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
doapi.dropletDestroy(droplet.id, function(body){
setTimeout(deleteDroplets, 1000, droplets);
});
}
doapi.dropletsByTag(tag, function(data){
data = JSON.parse(data);
console.log('current worker ids', currentIDs);
console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){
return item.name+' | '+item.id;
}));
deleteDroplets(data['droplets']);
});
};
workers.startRunners = function(args){
// console.log('starting runners on', args.worker.name, args.worker.ip)
if(!args.worker || workers.settings.image > args.worker.image.id){
console.log('blocked outdated worker', workers.settings.image, args.worker.image.id)
return ;
}
args.stopPercent = args.stopPercent || 80;
args.onStart = args.onStart || function(){};
args.onDone = args.onDone || function(){};
ramPercentUsed(args.worker.ip, function(usedMemPercent){
if(usedMemPercent > args.stopPercent ){
console.log('using', String(usedMemPercent).trim(),
'percent memory, stopping runner creation!', args.worker.availrunners.length,
'created on ', args.worker.name
);
args.onDone(args);
return ;
}
var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
// console.log('Free ram check passed!')
lxc.startEphemeral(name, 'crunner0', args.worker.ip, function(data){
if(!data.ip) return setTimeout(workers.startRunners, 0, args);
// console.log('started runner on', args.worker.name)
var runner = {
ip: data.ip,
name: name,
worker: args.worker,
label: args.worker.name + ':' + name
};
args.onStart(runner, args);
args.worker.availrunners.push(runner);
setTimeout(workers.startRunners, 0, args);
});
});
};
workers.startRunners = function(worker, newWorker, stopPercent){
// console.log('starting runners on', worker.name, worker.ip)
stopPercent = stopPercent || 80;
ramPercentUsed(worker.ip, function(usedMemPercent){
if(usedMemPercent < stopPercent ){
var name = 'crunner-'+(Math.random()*100).toString().slice(-4);
return lxc.startEphemeral(name, 'crunner0', worker.ip, function(data){
if(!data.ip) return setTimeout(workers.startRunners(worker, newWorker),0);
// console.log('started runner on', worker.name)
if(newWorker){
worker = workers[workers.push(worker)-1];
doapi.domianAddRecord({
domain: "codeland.us",
type: "A",
name: "*."+worker.name+".workers",
data: worker.publicIP
}, function(){});
}
workers.checkBalance = function(){
console.log('checking balance');
worker.availrunners.push({
ip: data.ip,
name: name,
worker: worker,
label: worker.name + ':' + name
});
return setTimeout(workers.startRunners(worker, false ,stopPercent), 0);
});
}else{
console.log('using', String(usedMemPercent), 'percent memory, stopping runner creation!', worker.availrunners.length, 'created on ', worker.name);
if(workers.length < workers.settings.min){
console.log('less then '+ workers.settings.min +' workers, starting a droplet');
for(var i=workers.settings.min-workers.length; i--;) workers.create();
return ;
}
for(let worker of workers){
console.log("checking", worker.name, "if zombie");
if(worker.availrunners.length === 0 && worker.usedrunner === 0){
workers.splice(workers.indexOf(worker), 1);
console.log('found zombie worker, destroying');
workers.destroy(worker);
}
}
if(workers.settings.minAvail > 0) return ;
var lastMinAval = 0;
for(let worker of workers.slice(-wokers.settings.minAvail)){
if(worker.usedrunner !== 0){
console.log('last 3 workers have no free runners, starting droplet');
return workers.create();
}
if(workers.length > workers.settings.min && workers.usedrunner === 0 ){
lastMinAval++;
}
}
if(lastMinAval == wokers.settings.minAvail){
console.log('Last 3 runners not used, killing last runner', workers.length);
return workers.destroy();
}
};
workers.settingsSave = function(){
jsonfile.writeFile('./workers.json', workers.settings, {spaces: 2}, function(err) {
console.error(err);
});
};
workers.checkBalance = function(){
var minWorkers = 1;
console.log('checking balance');
if(workers.length < minWorkers){
console.log('less then 3 workers, starting a droplet');
for(var i=minWorkers-workers.length; i--;) workers.create();
return ;
}
// if(workers[workers.length-3].usedrunner !== 0 && workers[workers.length-2].usedrunner !== 0 && workers[workers.length-1].usedrunner !== 0){
// console.log('last 3 workers have no free runners, starting droplet');
// return workers.create();
// }
// if(workers.length > minWorkers && workers[workers.length-3].usedrunner === 0 && workers[workers.length-2].usedrunner === 0 && workers[workers.length-1].usedrunner === 0){
// console.log('Last 2 runners not used, killing last runner', workers.length);
// return workers.destroy();
// }
// for(let worker of workers){
// if(worker.length <= 3) break;
// if(worker.availrunners.length === 0 && worker.usedrunner === 0){
// console.log('found zombie worker, destroying')
// workers.destroy(worker);
// }
// }
console.log('stopping workers balancing check');
workers.add = function(newWorkers){
newWorkers.forEach(function(worker){
workers.push(worker);
});
};
doapi.tagCreate('clwV'+workers.settings.version);
return workers;
})();
@ -196,6 +214,16 @@ var ramPercentUsed = function(ip, callback){
);
};
var runnerTimeout = function(runner, time){
time = time || 60000; // 1 minutes
if(runner.hasOwnProperty('timeout')){
clearTimeout(runner.timeout);
}
return runner.timeout = setTimeout(runnerFree, time, runner);
};
var runnerFree = function(runner){
lxc.stop(runner.name, runner.worker.ip);
runner.worker.usedrunner--;
@ -204,19 +232,7 @@ var runnerFree = function(runner){
}
delete label2runner[runner.label];
workers.startRunners(runner.worker);
};
var lxcTimeout = function(runner, time){
time = time || 60000; // 1 minutes
if(runner.hasOwnProperty('timeout')){
clearTimeout(runner.timeout);
}
return runner.timeout = setTimeout(function(){
runnerFree(runner);
}, time);
workers.startRunners({worker: runner.worker});
};
var getAvailrunner = function(runner){
@ -224,10 +240,11 @@ var getAvailrunner = function(runner){
if(worker.availrunners.length === 0) continue;
if(runner && runner.worker.index <= worker.index) break;
if(runner) runnerFree(runner);
return worker.getRunner();
}
if(runner) return runner;
return null;
};
var run = function(req, res, runner, count){
@ -240,10 +257,9 @@ var run = function(req, res, runner, count){
return res.json({error: 'No runners, try again soon.'});
}
if(count > 3){
if(count > 2){
console.log('to many reties on runner');
res.status(400);
return res.json({error: 'Runner restarted to many times'});
return res.status(400).json({error: 'Runner restarted to many times'});
}
var httpOptions = {
@ -256,86 +272,34 @@ var run = function(req, res, runner, count){
})
};
return request.post(httpOptions, function(error, response, body){
// console.log('runner response:', arguments)
if(error || response.statusCode !== 200) return run(req, res, getAvailrunner(), ++count);
body = JSON.parse(body);
body['ip'] = runner.label;
lxcTimeout(runner);
return res.json(body);
if(req.query.once){
res.json(body);
return runnerFree(runner, 0);
}
label2runner[runner.label] = runner;
body['ip'] = runner.label;
body['rname'] = runner.name;
body['wname'] = runner.worker.name;
res.json(body);
runnerTimeout(runner);
});
};
setTimeout(function(){
// console.log('Starting balance checking in 30 seconds')
setInterval(workers.checkBalance, 15000);
}, 180000);
setInterval(function(argument) {
workers.destroyOld();
}, 3600000); // 1 hour
}, 600000);
workers.destroyOld();
workers.checkBalance();
// router.get('/start/:name', function(req, res, next){
// return lxc.start(req.params.name, function(data){
// if(!data){
// return res.json({
// status: 500,
// name: req.params.name,
// message: data
// });
// }else{
// res.json({});
// }
// });
// });
// router.get('/live/:template/:name', function(req, res, next){
// return lxc.startEphemeral(req.params.name, req.params.template, function (data) {
// console.log('live', arguments);
// return res.json(data);
// });
// });
// router.get('/clone/:template/:name', function(req, res, next){
// return lxc.clone(req.params.name, req.params.template, function(data){
// console.log('clone', arguments);
// if( data.match(/Created runner/) ){
// return res.json({status: 200});
// }else{
// return res.json({status: 500, message: data});
// }
// });
// });
// router.get('/destroy/:name', function(req, res, next){
// return lxc.destroy(req.params.name, function(data){
// console.log('destroy', arguments);
// if(data){
// return res.json({status: 500, message: data});
// }else{
// return res.json({status: 200});
// }
// });
// });
// router.get('/info/:name', function(req, res, next){
// return lxc.info(req.params.name, function(data){
// return res.json(data);
// });
// });
// router.get('/list', function(req, res, next) {
// return lxc.list(workers.clworker0.ip, function(data){
// return res.json(data);
// });
// });
router.get('/stop/:name', function(req, res, next){
return lxc.stop(req.params.name, function(data){
@ -352,14 +316,89 @@ router.get('/destroyOld', function(req, res, next) {
res.send('?');
});
router.post('/updateID', function(req, res, next){
var newWorkers = {
workers: [],
image: req.query.image,
target: req.query.target || workers.length,
size: req.query.size || workers.settings.size,
version: workers.settings.version+1,
min: req.query.min || workers.settings,
minAvail: req.query.minAvail || workers.settings
};
doapi.tagCreate('clwV'+newWorkers.version);
workers.destroyOld('clwV'+newWorkers.version);
for(var i=0; i<newWorkers.target; i++){
doapi.dropletToActive({
name: 'clw'+newWorkers.version+'-'+(Math.random()*100).toString().slice(-4),
newWorkers: newWorkers,
image: newWorkers.image,
size: newWorkers.size,
onCreate: function(data, args){
doapi.dropletSetTag('clwV'+args.newWorkers.version, data.droplet.id);
},
onActive: function(droplet, args){
workers.startRunners({
worker: workers.makeWorkerObj(droplet),
newWorkers: args.newWorkers,
onStart: function(runner, args){
args.newWorkers.workers.push(args.worker);
console.log('onStart', args.worker.name);
args.onStart = function(){};
},
onDone: function(args){
console.log('new workers:', args.newWorkers.workers.length);
doapi.domianAddRecord({
domain: "codeland.us",
type: "A",
name: "*."+args.worker.name+".workers",
data: args.worker.publicIP
});
if(args.newWorkers.workers.length >= args.newWorkers.target){
console.log('upgrade complete!')
workers.settings.image = args.newWorkers.image;
workers.settings.size = args.newWorkers.size;
workers.settings.min = args.newWorkers.min;
workers.settings.minAvail = args.newWorkers.minAvail;
workers.forEach(function(worker){
worker.availrunners.forEach(function(runner){
lxc.stop(runner.name, runner.worker.ip);
});
worker.availrunners = [];
});
workers.add(args.newWorkers.workers);
workers.settingsSave();
workers.checkBalance();
}
}
});
}
});
}
res.json({status: "maybe?"});
});
router.get('/liststuff', function(req, res, next){
var obj = util.inspect(workers, {depth: 4});
res.send("<h1>Workers</h1><pre>"+obj+"</pre><h1>label2runner</h1><pre>"+util.inspect(label2runner)+'</pre><h1>DO calls</h1>'+doapi.calls);
});
router.post('/run/:ip?', function doRun(req, res, next){
router.get('/ping/:runner', function(req, res, next){
var runner = label2runner[req.params.runner];
runnerTimeout(runner);
res.json({res:''});
});
router.post('/run/:runner?', function (req, res, next){
console.log('hit runner route');
var runner = getAvailrunner(label2runner[req.params.ip]);
var runner = getAvailrunner(label2runner[req.params.runner]);
return run(req, res, runner);
});

1
routes/workers.json Normal file
View File

@ -0,0 +1 @@
{"image":"23505298","version":9,"lastSnapShotId":"18670778","size":"512mb","max":100,"min":3,"minAvail":3}

5
testAPI.py
View File

@ -7,8 +7,9 @@ def testAPI(times=100, sleep=2):
for i in range(times):
try:
res = r.post(
'http://codeland.bytedev.co:2000/api/run',
data={'code': 'pwd'}
'http://127.0.0.1:2000/api/run',
data={'code': 'pwd'},
params={'once': True}
)
if res.status_code != 200: errors += 1
print(i, res.status_code, res.content)

8
workers.json Normal file
View File

@ -0,0 +1,8 @@
{
"image": "23469057",
"version": 8,
"size": "512mb",
"max": 100,
"min": 3,
"minAvail": 3
}