Commit 9d5eee9d authored by Daniel Smith's avatar Daniel Smith

Refine custom file upload

parent 899af37f
......@@ -15,6 +15,7 @@
"author": "Daniel Smith <daniel.smith@qt.io>",
"license": "GPL-3.0",
"dependencies": {
"FormData": "^0.10.1",
"accepts": "^1.3.5",
"any-promise": "^1.3.0",
"array-flatten": "^2.1.2",
......@@ -22,6 +23,7 @@
"axios": "^0.18.0",
"batch": "^0.6.1",
"body-parser": "^1.18.3",
"buffer": "^5.4.0",
"bytes": "^3.1.0",
"child_process": "^1.0.2",
"content-disposition": "^0.5.3",
......@@ -29,6 +31,7 @@
"cookie": "^0.3.1",
"cookie-signature": "^1.1.0",
"debug": "^4.1.1",
"del": "^5.0.0",
"depd": "^2.0.0",
"destroy": "^1.0.4",
"diskusage": "^1.0.0",
......@@ -87,6 +90,7 @@
"serve-static": "^1.13.2",
"setprototypeof": "^1.1.1",
"statuses": "^1.5.0",
"temp-write": "^4.0.0",
"thenify": "^3.3.0",
"thenify-all": "^1.6.0",
"type-is": "^1.6.16",
......
......@@ -24,6 +24,9 @@ const readLastLines = require('read-last-lines');
var validator = require('email-validator');
const serveIndex = require('serve-index');
var updateLockout = false;
const tempWrite = require('temp-write');
const Buffer = require('buffer/').Buffer
const del = require('del');
// Declarations
var hosts = [];
......@@ -131,40 +134,43 @@ function addUpdateToQueue(req, res){
function updateJob(job){
// Search our arrays based on the type of request received. When the job is found, splice it out of it's array and place the updated job we got where it needs to go.
var tempjob;
if (job.type == 'running'){
var index = remoteQueuedJobs.findIndex(x => x.jobHash == job.job.jobHash);
if (index >= 0){
var tempjob = remoteQueuedJobs.splice(index, 1);
console.log(`spliced out ${tempjob[0].jobHash} from remoteQueued index ${index} when trying to remove ${job.job.jobHash}`);
tempjob = remoteQueuedJobs.splice(index, 1);
console.log(`Moved job ${tempjob[0].jobHash} from remote queued to remote running.`)
}
remoteRunningJobs.push(job.job);
} else if (job.type == 'finished' || job.type == 'cancelled'){
var index = remoteRunningJobs.findIndex(x => x.jobHash == job.job.jobHash);
if (index >= 0){
var tempjob = remoteRunningJobs.splice(index, 1);
console.log(`spliced out ${tempjob[0].jobHash} from remoteRunning index ${index} when trying to remove ${job.job.jobHash}`);
tempjob = remoteRunningJobs.splice(index, 1);
console.log(`Moved job ${tempjob[0].jobHash} from remote running to Finished.`);
} else {
index = remoteQueuedJobs.findIndex(x => x.jobHash == job.job.jobHash);
if (index >= 0){
var tempjob = remoteQueuedJobs.splice(index, 1);
console.log(`spliced out ${tempjob[0].jobHash} from remoteQueued index ${index} when trying to remove ${job.job.jobHash}`);
tempjob = remoteQueuedJobs.splice(index, 1);
} else {
console.log(`Received notification that job ${job.job.jobHash} was cancelled or finished, but we can't find it. This probably means that we missed an earlier update...`);
}
console.log(`Moved job ${tempjob[0].jobHash} from remote queued to Finished.`);
}
completedJobs.push(job.job);
} else if (job.type == 'cancelling'){
var index = remoteRunningJobs.findIndex(x => x.jobHash == job.job.jobHash);
if (index >= 0){
remoteRunningJobs[index].status = 'cancelling';
console.log(`Set remote running job ${remoteRunningJobs[index].jobHash} to cancelling when trying to set ${job.job.jobHash}`);
console.log(`Set remote running job ${remoteRunningJobs[index].jobHash} to cancelling.`);
} else {
index = remoteQueuedJobs.findIndex(x => x.jobHash == job.job.jobHash);
if (index >= 0){
remoteQueuedJobs[index].status = 'cancelling';
console.log(`Set remote queued job ${remoteQueuedJobs[index].jobHash} to cancelling when trying to set ${job.job.jobHash}`);
console.log(`Set remote queued job ${remoteQueuedJobs[index].jobHash} to cancelling.`);
} else {
console.log(`Received notification that job ${job.job.jobHash} was cancelled, but we can't find it. This probably means that we missed an earlier update...`);
}
......@@ -443,6 +449,18 @@ function runTest(test) {
// The job was in cancelling status when it exited. Set it to cancelled now.
test.status = 'cancelled';
}
// if(test.custom_benchmark_file){
// if(test.custom_benchmark_file.tempFilePath){
// (async () => {
// var re = /\\/gi;
// // can't use path.normalize here. del@5.0.0 module doesn't seem to work with windows style paths at the moment.
// const deletedPaths = await del([pathmodule.dirname(test.custom_benchmark_file.tempFilePath).replace(re, "/")], {force: true});
// console.log('Deleted files and directories:\n', deletedPaths.join('\n'));
// })(error => {console.log(error)});
// }
// }
notifyJob.emit('JobDone'); // Emit the JobDone signal to do some more tasks and prep for the next job in queue.
// saveTests(); //Backup the JSON file on disk for the status web page. // Not implemented!
......@@ -554,7 +572,7 @@ function scheduleJobRequest(req, res){
good_commit: req.body.good_commit,
openGLBackend: req.body.openGLBackend,
test_name: req.body.test_name,
custom_benchmark_file: req.body.custom_benchmark_file || req.files.custom_benchmark_file,
custom_benchmark_file: req.files ? req.files.custom_benchmark_file : req.body.custom_benchmark_file,
expected_regression: req.body.expected_regression,
test_all: req.body.test_all,
test_hosts: Array.isArray(req.body.test_hosts) ? [req.body.test_hosts[i]] : [req.body.test_hosts],
......@@ -563,6 +581,14 @@ function scheduleJobRequest(req, res){
master_host: req.body.master_host,
};
//Dump the data from the custom file to disk as a temp file.
if (job.custom_benchmark_file){
// Check to see if we have a properly formed data buffer. If not, re-create it from the object.
if (! (job.custom_benchmark_file.data.constructor && job.custom_benchmark_file.data.constructor.isBuffer && job.custom_benchmark_file.data.constructor(job.custom_benchmark_file.data))){
job.custom_benchmark_file.data = Buffer.from(job.custom_benchmark_file.data.data)
}
}
if (job.owner == ''){
job.owner = 'default'; // Set a display name if the email address field wasn't filled.
}
......@@ -570,6 +596,7 @@ function scheduleJobRequest(req, res){
if (req.body.test_hosts[i] == myname){
// Either we're a remote host and received a job, or we're the scheduler and have a job thread available.
console.log('executing test locally.');
job.custom_benchmark_file['tempFilePath'] = tempWrite.sync(job.custom_benchmark_file.data, `${job.custom_benchmark_file.name}`);
const jobHash = JSum.digest(job, 'SHA256', 'hex'); // Create a unique hash for this job
console.log(`Hash for this job: ${jobHash}`);
job['jobHash'] = jobHash;
......@@ -601,7 +628,11 @@ function scheduleJobRequest(req, res){
// We're processing the meta job request and there's a host that isn't the scheduler. Mark it as remote and forward it.
job['remote_job'] = true;
job['master_host'] = ip.address(); // Provide some info about where to send future job updates.
// delete job['custom_benchmark_file'];
// data = new FormData();
// data.extend(job);
// data.append('custom_benchmark_file', )
console.log(`Posting job to http://${hosts[hosts.findIndex(x => x.hostname == req.body.test_hosts[i])].ip}:${webPort}/remoteSchedule`);
axios.post(`http://${hosts[hosts.findIndex(x => x.hostname == req.body.test_hosts[i])].ip}:${webPort}/remoteSchedule`, job).then(
......@@ -1768,9 +1799,9 @@ function runServer(){
extended: true,
}));
server.use(fileUpload({
useTempFiles : true,
tempFileDir : os.tmpdir(),
preserveExtension : true
// useTempFiles : true,
// tempFileDir : os.tmpdir(),
// preserveExtension : true
}));
server.use(express.json()); // to support JSON-encoded bodies
server.use(express.urlencoded()); // to support URL-encoded bodies
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment