Browse Source

Much cleaner promise design

dev/git-series/gccdum
Matt Godbolt 8 years ago
parent
commit
71dda61541
  1. 192
      app.js
  2. 244
      lib/compile.js
  3. 2
      package.json

192
app.js

@ -27,12 +27,10 @@
var nopt = require('nopt'),
os = require('os'),
props = require('./lib/properties'),
compileHandler = require('./lib/compile').compileHandler,
express = require('express'),
child_process = require('child_process'),
temp = require('temp'),
path = require('path'),
async = require('async'),
LRU = require('lru-cache'),
fs = require('fs-extra'),
Promise = require('promise');
@ -51,188 +49,6 @@ var rootDir = opts.rootDir || './etc';
props.initialize(rootDir + '/config', propHierarchy);
var port = props.get('gcc-explorer', 'port', 10240);
var compilersByExe = {};
var compilerExecutables;
var cache = LRU({
max: props.get('gcc-explorer', 'cacheMb') * 1024 * 1024,
length: function (n) {
return n.length;
}
});
var cacheHits = 0;
var cacheMisses = 0;
var compileQueue = async.queue(function (task, callback) {
console.log("Compiling, queue size " + compileQueue.length());
task.task(callback);
}, props.get("gcc-explorer", "maxConcurrentCompiles", 1));
compileQueue.drain = function () {
console.log("Compile queue empty");
};
compileQueue.saturated = function () {
console.log("Compile queue full");
};
function checkOptions(options) {
var okOptions = new RegExp(props.get('gcc-options', 'whitelistRe', '.*'));
var badOptions = new RegExp(props.get('gcc-options', 'blacklistRe'));
var error = [];
options.forEach(function (option) {
if (!option.match(okOptions) || option.match(badOptions)) {
error.push(option);
}
});
if (error.length > 0) return "Bad options: " + error.join(", ");
return null;
}
function checkSource(source) {
var re = /^\s*#include(_next)?\s+["<"](\/|.*\.\.)/;
var failed = [];
source.split('\n').forEach(function (line, index) {
if (line.match(re)) {
failed.push("<stdin>:" + (index + 1) + ":1: no absolute or relative includes please");
}
});
if (failed.length > 0) return failed.join("\n");
return null;
}
function cacheStats() {
console.log("Cache stats: " + cacheHits + " hits, " + cacheMisses + " misses");
}
function compileHandler(compilers) {
var compilersByExe = {};
compilers.forEach(function (compiler) {
compilersByExe[compiler.exe] = compiler; // todo: kill global
});
return function compile(req, res) {
var source = req.body.source;
var compiler = req.body.compiler;
if (getCompilerExecutables().indexOf(compiler) < 0) {
return res.end(JSON.stringify({code: -1, stderr: "bad compiler " + compiler}));
}
var compilerInfo = compilersByExe[compiler];
if (!compilerInfo) {
return res.end(JSON.stringify({code: -1, stderr: "bad compiler " + compiler}));
}
var options = req.body.options.split(' ').filter(function (x) {
return x !== "";
});
var filters = req.body.filters;
var optionsErr = checkOptions(options);
if (optionsErr) {
return res.end(JSON.stringify({code: -1, stderr: optionsErr}));
}
var sourceErr = checkSource(source);
if (sourceErr) {
return res.end(JSON.stringify({code: -1, stderr: sourceErr}));
}
var key = compiler + " | " + source + " | " + options + " | " + filters.intel;
var cached = cache.get(key);
if (cached) {
cacheHits++;
cacheStats();
res.end(cached);
return;
}
cacheMisses++;
var compileTask = function (taskFinished) {
temp.mkdir('gcc-explorer-compiler', function (err, dirPath) {
if (err) {
return res.end(JSON.stringify({code: -1, stderr: "Unable to open temp file: " + err}));
}
var postProcess = props.get("gcc-explorer", "postProcess");
var inputFilename = path.join(dirPath, props.get("gcc-explorer", "compileFilename"));
var outputFilename = path.join(dirPath, 'output.S');
if (compilerInfo.supportedOpts['-masm']) {
var syntax = '-masm=att'; // default at&t
if (filters.intel == "true") syntax = '-masm=intel';
options = options.concat([syntax]);
}
var compileToAsm = props.get("gcc-explorer", "compileToAsm", "-S").split(" ");
options = options.concat(['-g', '-o', outputFilename]).concat(compileToAsm).concat([inputFilename]);
var file = fs.openSync(inputFilename, "w");
fs.writeSync(file, source);
fs.closeSync(file);
var okToCache = true;
var compilerWrapper = props.get("gcc-explorer", "compiler-wrapper");
if (compilerWrapper) {
options = [compiler].concat(options);
compiler = compilerWrapper;
}
var child = child_process.spawn(
compiler,
options,
{detached: true}
);
var stdout = "";
var stderr = "";
var timeout = setTimeout(function () {
okToCache = false;
child.kill();
stderr += "\nKilled - processing time exceeded";
}, props.get("gcc-explorer", "compileTimeoutMs", 100));
child.stdout.on('data', function (data) {
stdout += data;
});
child.stderr.on('data', function (data) {
stderr += data;
});
child.on('exit', function (code) {
clearTimeout(timeout);
var maxSize = props.get("gcc-explorer", "max-asm-size", 8 * 1024 * 1024);
function complete(data) {
var result = JSON.stringify({
stdout: stdout,
stderr: stderr,
asm: data,
code: code
});
if (okToCache) {
cache.set(key, result);
cacheStats();
}
res.end(result);
fs.remove(dirPath);
taskFinished();
}
if (code !== 0) {
return complete("<Compilation failed>");
}
try {
var size = fs.statSync(outputFilename).size;
if (size >= maxSize) {
return complete("<No output: generated assembly was too large (" + size + " > " + maxSize + " bytes)>");
}
} catch (e) {
return complete("<No output file>");
}
child_process.exec('cat "' + outputFilename + '" | ' + postProcess,
{maxBuffer: maxSize},
function (err, filt_stdout, filt_stderr) {
var data = filt_stdout;
if (err) {
if ("")
data = '<No output: ' + err + '>';
}
complete(data);
});
});
child.stdin.end();
});
};
compileQueue.push({task: compileTask});
};
}
function loadSources() {
var sourcesDir = "lib/sources";
@ -288,9 +104,6 @@ function getSource(req, res, next) {
}
function getCompilerExecutables() {
if (compilerExecutables) {
return compilerExecutables;
}
var exes = props.get("gcc-explorer", "compilers", "/usr/bin/g++").split(":");
var ndk = props.get('gcc-explorer', 'androidNdk');
if (ndk) {
@ -311,7 +124,6 @@ function getCompilerExecutables() {
});
exes.push.apply(exes, toolchains);
}
compilerExecutables = exes;
return exes;
}
@ -395,4 +207,6 @@ findCompilers().then(function (compilers) {
console.log("Listening on http://" + os.hostname() + ":" + port + "/");
console.log("=======================================");
webServer.listen(port);
}).catch(function (err) {
console.log("Error: " + err.stack);
});

244
lib/compile.js

@ -0,0 +1,244 @@
// Copyright (c) 2012-2015, Matt Godbolt
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
var props = require('./properties'),
child_process = require('child_process'),
temp = require('temp'),
path = require('path'),
LRU = require('lru-cache'),
fs = require('fs-extra'),
Promise = require('promise'),
Queue = require('promise-queue');
Queue.configure(Promise);
temp.track();
setInterval(function () {
temp.cleanup(function (err) {
if (err) throw new Error(err);
});
}, props.get('gcc-explorer', 'secsBetweenCleanups', 60) * 1000);
function Compile(compilers) {
this.compilersByExe = {};
var self = this;
compilers.forEach(function (compiler) {
self.compilersByExe[compiler.exe] = compiler;
});
this.okOptions = new RegExp(props.get('gcc-options', 'whitelistRe', '.*'));
this.badOptions = new RegExp(props.get('gcc-options', 'blacklistRe'));
this.cache = LRU({
max: props.get('gcc-explorer', 'cacheMb') * 1024 * 1024,
length: function (n) {
return n.length;
}
});
this.cacheHits = 0;
this.cacheMisses = 0;
this.compileQueue = new Queue(props.get("gcc-explorer", "maxConcurrentCompiles", 1), Infinity);
}
Compile.prototype.newTempDir = function () {
return new Promise(function (resolve, reject) {
temp.mkdir('gcc-explorer-compiler', function (err, dirPath) {
if (err)
reject("Unable to open temp file: " + err);
else
resolve(dirPath);
});
});
};
Compile.prototype.writeFile = Promise.denodeify(fs.writeFile);
Compile.prototype.stat = Promise.denodeify(fs.stat);
Compile.prototype.runCompiler = function(compiler, options) {
var okToCache = true;
var child = child_process.spawn(
compiler,
options,
{detached: true}
);
var stdout = "";
var stderr = "";
var timeout = setTimeout(function () {
okToCache = false;
child.kill();
stderr += "\nKilled - processing time exceeded";
}, props.get("gcc-explorer", "compileTimeoutMs", 100));
child.stdout.on('data', function (data) {
stdout += data;
});
child.stderr.on('data', function (data) {
stderr += data;
});
return new Promise(function (resolve) {
child.on('exit', function (code) {
clearTimeout(timeout);
resolve({code: code, stdout: stdout, stderr: stderr, okToCache: okToCache});
});
child.stdin.end();
});
};
Compile.prototype.compile = function (source, compiler, options, filters) {
var self = this;
return new Promise(function (resolve, reject) {
var optionsError = self.checkOptions(options);
if (optionsError) return reject(optionsError);
var sourceError = self.checkSource(source);
if (sourceError) return reject(sourceError);
var compilerInfo = self.compilersByExe[compiler];
if (!compilerInfo) {
return reject("Bad compiler " + compiler);
}
var key = compiler + " | " + source + " | " + options + " | " + filters.intel;
var cached = self.cache.get(key);
if (cached) {
self.cacheHits++;
self.cacheStats();
return resolve(cached);
}
self.cacheMisses++;
var queuedPromise = self.compileQueue.add(function () {
return self.newTempDir().then(function (dirPath) {
var inputFilename = path.join(dirPath, props.get("gcc-explorer", "compileFilename"));
return self.writeFile(inputFilename, source).then(function () {
return {inputFilename: inputFilename, dirPath: dirPath};
});
}).then(function (info) {
var inputFilename = info.inputFilename;
var dirPath = info.dirPath;
var postProcess = props.get("gcc-explorer", "postProcess");
var outputFilename = path.join(dirPath, 'output.S');
if (compilerInfo.supportedOpts['-masm']) {
var syntax = '-masm=att'; // default at&t
if (filters.intel == "true") syntax = '-masm=intel';
options = options.concat([syntax]);
}
var compileToAsm = props.get("gcc-explorer", "compileToAsm", "-S").split(" ");
options = options.concat(['-g', '-o', outputFilename]).concat(compileToAsm).concat([inputFilename]);
var compilerWrapper = props.get("gcc-explorer", "compiler-wrapper");
if (compilerWrapper) {
options = [compiler].concat(options);
compiler = compilerWrapper;
}
var maxSize = props.get("gcc-explorer", "max-asm-size", 8 * 1024 * 1024);
var gotAsmFinished = self.runCompiler(compiler, options).then(function (result) {
if (result.code !== 0) {
result.asm = "<Compilation failed>";
return result;
}
return self.stat(outputFilename).then(function (stat) {
if (stat.size >= maxSize) {
result.asm = "<No output: generated assembly was too large (" + size + " > " + maxSize + " bytes)>";
return result;
}
return new Promise(function (resolve) {
child_process.exec('cat "' + outputFilename + '" | ' + postProcess,
{maxBuffer: maxSize},
function (err, data) {
if (err)
data = '<No output: ' + err + '>';
result.asm = data;
resolve(result);
});
});
}, function () {
result.asm = "<No output file>";
return result;
});
});
return gotAsmFinished.then(function (result) {
if (result.okToCache) {
self.cache.set(key, result);
self.cacheStats();
}
return result;
});
});
});
return queuedPromise.then(resolve, reject);
});
};
Compile.prototype.checkOptions = function (options) {
var error = [];
var self = this;
options.forEach(function (option) {
if (!option.match(self.okOptions) || option.match(self.badOptions)) {
error.push(option);
}
});
if (error.length > 0) return "Bad options: " + error.join(", ");
return null;
};
Compile.prototype.checkSource = function (source) {
var re = /^\s*#include(_next)?\s+["<"](\/|.*\.\.)/;
var failed = [];
source.split('\n').forEach(function (line, index) {
if (line.match(re)) {
failed.push("<stdin>:" + (index + 1) + ":1: no absolute or relative includes please");
}
});
if (failed.length > 0) return failed.join("\n");
return null;
};
Compile.prototype.cacheStats = function () {
console.log("Cache stats: " + this.cacheHits + " hits, " + this.cacheMisses + " misses");
};
function compileHandler(compilers) {
var compileObj = new Compile(compilers);
return function compile(req, res) {
var source = req.body.source;
var compiler = req.body.compiler;
var options = req.body.options.split(' ').filter(function (x) {
return x !== "";
});
var filters = req.body.filters;
compileObj.compile(source, compiler, options, filters).then(
function (result) {
res.end(JSON.stringify(result));
},
function (error) {
if (typeof(error) !== "string") {
error = "Internal GCC explorer error: " + error.toString();
}
res.end(JSON.stringify({code: -1, stderr: error}));
}
);
};
}
module.exports = {
compileHandler: compileHandler
};

2
package.json

@ -16,13 +16,13 @@
},
"main": "./app.js",
"dependencies": {
"async": "0.9.x",
"body-parser": "1.10.x",
"express": "4.11.x",
"fs-extra": "0.8.x",
"lru-cache": "2.5.x",
"morgan": "1.5.x",
"promise": "6.1.x",
"promise-queue": "2.1.x",
"nopt": "3.0.x",
"serve-favicon": "2.2.x",
"serve-static": "1.8.x",

Loading…
Cancel
Save