fixes
This commit is contained in:
parent
128320f80e
commit
cfd6c373ad
@ -64,10 +64,10 @@ var workers = (function(){
|
|||||||
args.onStart = function(){};
|
args.onStart = function(){};
|
||||||
},
|
},
|
||||||
onDone: function(args){
|
onDone: function(args){
|
||||||
console.log("done with runners on", worker.name);
|
console.log("Seeded runners on", worker.name);
|
||||||
|
workers.currentCreating--;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
workers.currentCreating--;
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -112,7 +112,7 @@ var workers = (function(){
|
|||||||
|
|
||||||
var worker = worker || workers.pop();
|
var worker = worker || workers.pop();
|
||||||
return doapi.dropletDestroy(worker.id, function(body) {
|
return doapi.dropletDestroy(worker.id, function(body) {
|
||||||
console.log('body of destroy', body);
|
console.log('Deleted worker', worker.name);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -120,24 +120,26 @@ var workers = (function(){
|
|||||||
// Delete works that with
|
// Delete works that with
|
||||||
|
|
||||||
tag = tag || tagPrefix + workers.settings.version;
|
tag = tag || tagPrefix + workers.settings.version;
|
||||||
var currentIDs = workers.__workersId();
|
let currentIDs = workers.__workersId();
|
||||||
|
|
||||||
var deleteDroplets = function(droplets){
|
let deleteDroplets = function(droplets){
|
||||||
if(droplets.length === 0) return true;
|
if(droplets.length === 0) return true;
|
||||||
var droplet = droplets.pop();
|
let droplet = droplets.pop();
|
||||||
if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
|
if(~currentIDs.indexOf(droplet.id)) return deleteDroplets(droplets);
|
||||||
|
|
||||||
doapi.dropletDestroy(droplet.id, function(body){
|
doapi.dropletDestroy(droplet.id, function(body){
|
||||||
setTimeout(deleteDroplets, 1000, droplets);
|
setTimeout(deleteDroplets, 1000, droplets);
|
||||||
|
if(!droplets.length) console.log(`Finished deleting workers tagged ${tag}.`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
doapi.dropletsByTag(tag, function(data){
|
doapi.dropletsByTag(tag, function(data){
|
||||||
data = JSON.parse(data);
|
data = JSON.parse(data);
|
||||||
console.log('current worker ids', currentIDs);
|
console.log(`Deleting ${data['droplets'].length} workers tagged ${tag}. Workers`,
|
||||||
console.log('do droplets', data['droplets'].length, data['droplets'].map(function(item){
|
data['droplets'].map(function(item){
|
||||||
return item.name+' | '+item.id;
|
return item.name+' | '+item.id;
|
||||||
}));
|
})
|
||||||
|
);
|
||||||
|
|
||||||
deleteDroplets(data['droplets']);
|
deleteDroplets(data['droplets']);
|
||||||
});
|
});
|
||||||
@ -148,7 +150,7 @@ var workers = (function(){
|
|||||||
|
|
||||||
// dont make runners on out dated workers
|
// dont make runners on out dated workers
|
||||||
if(!args.worker || workers.settings.image > args.worker.image.id){
|
if(!args.worker || workers.settings.image > args.worker.image.id){
|
||||||
console.log('blocked outdated worker', workers.settings.image, args.worker.image.id)
|
console.log(`Blocked outdated worker(${args.worker.image.id}), current image ${workers.settings.image}.`)
|
||||||
return ;
|
return ;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -194,13 +196,13 @@ var workers = (function(){
|
|||||||
let zombies = 0;
|
let zombies = 0;
|
||||||
|
|
||||||
for(let worker of workers){
|
for(let worker of workers){
|
||||||
console.log("checking", worker.name, "if zombie");
|
console.log(`Checking if ${worker.name} is a zombie worker.`);
|
||||||
// if a runner has no available runners and no used runners, its a
|
// if a runner has no available runners and no used runners, its a
|
||||||
// zombie. This should happen when a newer image ID has been added
|
// zombie. This should happen when a newer image ID has been added
|
||||||
// and old workers slowly lose there usefulness.
|
// and old workers slowly lose there usefulness.
|
||||||
if(worker.availrunners.length === 0 && worker.usedrunners === 0){
|
if(worker.availrunners.length === 0 && worker.usedrunners === 0){
|
||||||
workers.splice(workers.indexOf(worker), 1);
|
workers.splice(workers.indexOf(worker), 1);
|
||||||
console.log('found zombie worker, destroying');
|
console.log(`Zombie! Worker ${worker.name}, destroying.`);
|
||||||
workers.destroy(worker);
|
workers.destroy(worker);
|
||||||
zombies++;
|
zombies++;
|
||||||
}
|
}
|
||||||
@ -210,15 +212,14 @@ var workers = (function(){
|
|||||||
};
|
};
|
||||||
|
|
||||||
workers.checkBalance = function(){
|
workers.checkBalance = function(){
|
||||||
console.log('checking balance');
|
console.log(`Checking balance.`);
|
||||||
|
|
||||||
workers.checkForZombies();
|
workers.checkForZombies();
|
||||||
|
|
||||||
console.log(workers.currentCreating+workers.length, workers.currentCreating+workers.length < workers.min)
|
|
||||||
// if there are workers being created, stop scale up and down check
|
// if there are workers being created, stop scale up and down check
|
||||||
if(workers.currentCreating+workers.length < workers.settings.min) null;
|
if(workers.currentCreating+workers.length < workers.settings.min) null;
|
||||||
else if(workers.currentCreating)
|
else if(workers.currentCreating)
|
||||||
return console.log('killing balance, worker is being created.');
|
return console.log(`Killing balance, workers are being created.`);
|
||||||
|
|
||||||
// hold amount of workers with no used runners
|
// hold amount of workers with no used runners
|
||||||
var lastMinAval = 0;
|
var lastMinAval = 0;
|
||||||
@ -323,16 +324,16 @@ var getAvailrunner = function(runner){
|
|||||||
|
|
||||||
var run = function(req, res, runner, count){
|
var run = function(req, res, runner, count){
|
||||||
count = count || 0;
|
count = count || 0;
|
||||||
console.log('run start', count);
|
console.log(`Runner starting attempt ${count}.`);
|
||||||
|
|
||||||
if(!runner){
|
if(!runner){
|
||||||
console.log('no runner');
|
console.log(`No runner available!`);
|
||||||
res.status(503);
|
res.status(503);
|
||||||
return res.json({error: 'No runners, try again soon.'});
|
return res.json({error: 'No runners, try again soon.'});
|
||||||
}
|
}
|
||||||
|
|
||||||
if(count > 2){
|
if(count > 2){
|
||||||
console.log('to many reties on runner');
|
console.log(`Runner attempt failed, to many requests!`);
|
||||||
return res.status(400).json({error: 'Runner restarted to many times'});
|
return res.status(400).json({error: 'Runner restarted to many times'});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -366,10 +367,9 @@ var run = function(req, res, runner, count){
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
console.log('========STARTING===========')
|
||||||
setInterval(workers.checkBalance, 15000);
|
setInterval(workers.checkBalance, 15000);
|
||||||
|
|
||||||
workers.destroyByTag();
|
workers.destroyByTag();
|
||||||
workers.checkBalance();
|
|
||||||
|
|
||||||
|
|
||||||
router.get('/stop/:name', function(req, res, next){
|
router.get('/stop/:name', function(req, res, next){
|
||||||
@ -458,7 +458,14 @@ router.post('/updateID', function(req, res, next){
|
|||||||
|
|
||||||
router.get('/liststuff', function(req, res, next){
|
router.get('/liststuff', function(req, res, next){
|
||||||
var obj = util.inspect(workers, {depth: 4});
|
var obj = util.inspect(workers, {depth: 4});
|
||||||
res.send("<h1>Workers</h1><pre>"+obj+"</pre><h1>label2runner</h1><pre>"+util.inspect(label2runner)+'</pre><h1>DO calls</h1>'+doapi.calls);
|
res.send(`
|
||||||
|
<h1>Workers</h1>
|
||||||
|
<pre>${obj}</pre>
|
||||||
|
<h1>label2runner</h1>
|
||||||
|
<pre>${util.inspect(label2runner)}</pre>
|
||||||
|
<h1>DO calls</h1>
|
||||||
|
${doapi.calls}
|
||||||
|
`);
|
||||||
});
|
});
|
||||||
|
|
||||||
router.get('/ping/:runner', function(req, res, next){
|
router.get('/ping/:runner', function(req, res, next){
|
||||||
@ -468,7 +475,7 @@ router.get('/ping/:runner', function(req, res, next){
|
|||||||
});
|
});
|
||||||
|
|
||||||
router.post('/run/:runner?', function (req, res, next){
|
router.post('/run/:runner?', function (req, res, next){
|
||||||
console.log('hit runner route');
|
console.log(`Request runner route!`);
|
||||||
var runner = getAvailrunner(label2runner[req.params.runner]);
|
var runner = getAvailrunner(label2runner[req.params.runner]);
|
||||||
return run(req, res, runner);
|
return run(req, res, runner);
|
||||||
});
|
});
|
||||||
|
61
testAPI.js
61
testAPI.js
@ -1,32 +1,59 @@
|
|||||||
const request = require('request');
|
const request = require('request');
|
||||||
|
|
||||||
var till = 5;
|
var callRunner = (function(){
|
||||||
var completed = 0;
|
let started = 0;
|
||||||
var errors = 0;
|
let completed = 0;
|
||||||
|
let errors = 0;
|
||||||
|
let noRunner = 0;
|
||||||
|
|
||||||
|
let func = function(code, callback) {
|
||||||
|
// code | `string` block of code to send runner or
|
||||||
|
// `number` sleep timeout in seconds
|
||||||
|
let sleepTime = 0;
|
||||||
|
let id = started++;
|
||||||
|
callback = callback || function(){};
|
||||||
|
|
||||||
|
if(/^\+?(0|[1-9]\d*)$/.test(code)){
|
||||||
|
sleepTime = code;
|
||||||
|
code = null;
|
||||||
|
}
|
||||||
|
console.log(id, ': Running...');
|
||||||
|
|
||||||
|
|
||||||
for(let i=0; i<till; i++){
|
|
||||||
console.log('calling', i);
|
|
||||||
let httpOptions = {
|
let httpOptions = {
|
||||||
url: 'http://codeland.bytedev.co:2000/api/run?once=true',
|
url: 'http://codeland.bytedev.co:2000/api/run?once=true',
|
||||||
form: {
|
form: {
|
||||||
code: `python3 -c "
|
code: code || `python3 -c "
|
||||||
from time import sleep
|
from time import sleep
|
||||||
sleep(10)
|
sleep(${sleepTime})
|
||||||
"`,
|
"`,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
request.post(httpOptions, function(error, response, body){
|
|
||||||
|
return request.post(httpOptions, function(error, response, body){
|
||||||
completed++;
|
completed++;
|
||||||
if(error || response.statusCode !== 200){
|
let res = ``;
|
||||||
|
if(response.statusCode == 503){
|
||||||
|
noRunner++;
|
||||||
|
}else if(error || response.statusCode !== 200){
|
||||||
errors++;
|
errors++;
|
||||||
}
|
|
||||||
|
|
||||||
body = JSON.parse(body);
|
body = JSON.parse(body);
|
||||||
let res = (Buffer.from(body.res, 'base64').toString('ascii'));
|
res = (Buffer.from(body.res, 'base64').toString('ascii'));
|
||||||
console.log(i, res);
|
|
||||||
if(completed===till){
|
|
||||||
console.log(errors);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
console.log(`${id} with results ${res}. Errors ${errors}. No runner ${noRunner}. Completed ${completed}`);
|
||||||
|
|
||||||
|
callback()
|
||||||
|
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return func;
|
||||||
|
})();
|
||||||
|
|
||||||
|
|
||||||
|
let __do = function(till){
|
||||||
|
callRunner(String(Math.random())[3]);
|
||||||
|
setTimeout(__do, 1000, --till);
|
||||||
|
};
|
||||||
|
|
||||||
|
__do(50)
|
Loading…
x
Reference in New Issue
Block a user