From 71dda61541e86c09d513ac76e2501bd0f5bb4b16 Mon Sep 17 00:00:00 2001 From: Matt Godbolt Date: Tue, 20 Jan 2015 23:44:54 -0600 Subject: [PATCH] Much cleaner promise design --- app.js | 192 +------------------------------------- lib/compile.js | 244 +++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 3 files changed, 248 insertions(+), 190 deletions(-) create mode 100644 lib/compile.js diff --git a/app.js b/app.js index 792a7ff0..93990328 100755 --- a/app.js +++ b/app.js @@ -27,12 +27,10 @@ var nopt = require('nopt'), os = require('os'), props = require('./lib/properties'), + compileHandler = require('./lib/compile').compileHandler, express = require('express'), child_process = require('child_process'), - temp = require('temp'), path = require('path'), - async = require('async'), - LRU = require('lru-cache'), fs = require('fs-extra'), Promise = require('promise'); @@ -51,188 +49,6 @@ var rootDir = opts.rootDir || './etc'; props.initialize(rootDir + '/config', propHierarchy); var port = props.get('gcc-explorer', 'port', 10240); -var compilersByExe = {}; -var compilerExecutables; - -var cache = LRU({ - max: props.get('gcc-explorer', 'cacheMb') * 1024 * 1024, - length: function (n) { - return n.length; - } -}); -var cacheHits = 0; -var cacheMisses = 0; - -var compileQueue = async.queue(function (task, callback) { - console.log("Compiling, queue size " + compileQueue.length()); - task.task(callback); -}, props.get("gcc-explorer", "maxConcurrentCompiles", 1)); -compileQueue.drain = function () { - console.log("Compile queue empty"); -}; -compileQueue.saturated = function () { - console.log("Compile queue full"); -}; - -function checkOptions(options) { - var okOptions = new RegExp(props.get('gcc-options', 'whitelistRe', '.*')); - var badOptions = new RegExp(props.get('gcc-options', 'blacklistRe')); - var error = []; - options.forEach(function (option) { - if (!option.match(okOptions) || option.match(badOptions)) { - error.push(option); - } - }); - if (error.length > 0) return "Bad options: " + error.join(", "); - return null; -} - -function checkSource(source) { - var re = /^\s*#include(_next)?\s+["<"](\/|.*\.\.)/; - var failed = []; - source.split('\n').forEach(function (line, index) { - if (line.match(re)) { - failed.push(":" + (index + 1) + ":1: no absolute or relative includes please"); - } - }); - if (failed.length > 0) return failed.join("\n"); - return null; -} - -function cacheStats() { - console.log("Cache stats: " + cacheHits + " hits, " + cacheMisses + " misses"); -} - -function compileHandler(compilers) { - var compilersByExe = {}; - compilers.forEach(function (compiler) { - compilersByExe[compiler.exe] = compiler; // todo: kill global - }); - return function compile(req, res) { - var source = req.body.source; - var compiler = req.body.compiler; - if (getCompilerExecutables().indexOf(compiler) < 0) { - return res.end(JSON.stringify({code: -1, stderr: "bad compiler " + compiler})); - } - var compilerInfo = compilersByExe[compiler]; - if (!compilerInfo) { - return res.end(JSON.stringify({code: -1, stderr: "bad compiler " + compiler})); - } - var options = req.body.options.split(' ').filter(function (x) { - return x !== ""; - }); - var filters = req.body.filters; - var optionsErr = checkOptions(options); - if (optionsErr) { - return res.end(JSON.stringify({code: -1, stderr: optionsErr})); - } - var sourceErr = checkSource(source); - if (sourceErr) { - return res.end(JSON.stringify({code: -1, stderr: sourceErr})); - } - - var key = compiler + " | " + source + " | " + options + " | " + filters.intel; - var cached = cache.get(key); - if (cached) { - cacheHits++; - cacheStats(); - res.end(cached); - return; - } - cacheMisses++; - - var compileTask = function (taskFinished) { - temp.mkdir('gcc-explorer-compiler', function (err, dirPath) { - if (err) { - return res.end(JSON.stringify({code: -1, stderr: "Unable to open temp file: " + err})); - } - var postProcess = props.get("gcc-explorer", "postProcess"); - var inputFilename = path.join(dirPath, props.get("gcc-explorer", "compileFilename")); - var outputFilename = path.join(dirPath, 'output.S'); - if (compilerInfo.supportedOpts['-masm']) { - var syntax = '-masm=att'; // default at&t - if (filters.intel == "true") syntax = '-masm=intel'; - options = options.concat([syntax]); - } - var compileToAsm = props.get("gcc-explorer", "compileToAsm", "-S").split(" "); - options = options.concat(['-g', '-o', outputFilename]).concat(compileToAsm).concat([inputFilename]); - var file = fs.openSync(inputFilename, "w"); - fs.writeSync(file, source); - fs.closeSync(file); - var okToCache = true; - var compilerWrapper = props.get("gcc-explorer", "compiler-wrapper"); - if (compilerWrapper) { - options = [compiler].concat(options); - compiler = compilerWrapper; - } - var child = child_process.spawn( - compiler, - options, - {detached: true} - ); - var stdout = ""; - var stderr = ""; - var timeout = setTimeout(function () { - okToCache = false; - child.kill(); - stderr += "\nKilled - processing time exceeded"; - }, props.get("gcc-explorer", "compileTimeoutMs", 100)); - child.stdout.on('data', function (data) { - stdout += data; - }); - child.stderr.on('data', function (data) { - stderr += data; - }); - child.on('exit', function (code) { - clearTimeout(timeout); - var maxSize = props.get("gcc-explorer", "max-asm-size", 8 * 1024 * 1024); - - function complete(data) { - var result = JSON.stringify({ - stdout: stdout, - stderr: stderr, - asm: data, - code: code - }); - if (okToCache) { - cache.set(key, result); - cacheStats(); - } - res.end(result); - fs.remove(dirPath); - taskFinished(); - } - - if (code !== 0) { - return complete(""); - } - try { - var size = fs.statSync(outputFilename).size; - if (size >= maxSize) { - return complete(" " + maxSize + " bytes)>"); - } - } catch (e) { - return complete(""); - } - - child_process.exec('cat "' + outputFilename + '" | ' + postProcess, - {maxBuffer: maxSize}, - function (err, filt_stdout, filt_stderr) { - var data = filt_stdout; - if (err) { - if ("") - data = ''; - } - complete(data); - }); - }); - child.stdin.end(); - }); - }; - - compileQueue.push({task: compileTask}); - }; -} function loadSources() { var sourcesDir = "lib/sources"; @@ -288,9 +104,6 @@ function getSource(req, res, next) { } function getCompilerExecutables() { - if (compilerExecutables) { - return compilerExecutables; - } var exes = props.get("gcc-explorer", "compilers", "/usr/bin/g++").split(":"); var ndk = props.get('gcc-explorer', 'androidNdk'); if (ndk) { @@ -311,7 +124,6 @@ function getCompilerExecutables() { }); exes.push.apply(exes, toolchains); } - compilerExecutables = exes; return exes; } @@ -395,4 +207,6 @@ findCompilers().then(function (compilers) { console.log("Listening on http://" + os.hostname() + ":" + port + "/"); console.log("======================================="); webServer.listen(port); +}).catch(function (err) { + console.log("Error: " + err.stack); }); diff --git a/lib/compile.js b/lib/compile.js new file mode 100644 index 00000000..d7829359 --- /dev/null +++ b/lib/compile.js @@ -0,0 +1,244 @@ +// Copyright (c) 2012-2015, Matt Godbolt +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +var props = require('./properties'), + child_process = require('child_process'), + temp = require('temp'), + path = require('path'), + LRU = require('lru-cache'), + fs = require('fs-extra'), + Promise = require('promise'), + Queue = require('promise-queue'); + +Queue.configure(Promise); +temp.track(); + +setInterval(function () { + temp.cleanup(function (err) { + if (err) throw new Error(err); + }); +}, props.get('gcc-explorer', 'secsBetweenCleanups', 60) * 1000); + +function Compile(compilers) { + this.compilersByExe = {}; + var self = this; + compilers.forEach(function (compiler) { + self.compilersByExe[compiler.exe] = compiler; + }); + this.okOptions = new RegExp(props.get('gcc-options', 'whitelistRe', '.*')); + this.badOptions = new RegExp(props.get('gcc-options', 'blacklistRe')); + this.cache = LRU({ + max: props.get('gcc-explorer', 'cacheMb') * 1024 * 1024, + length: function (n) { + return n.length; + } + }); + this.cacheHits = 0; + this.cacheMisses = 0; + this.compileQueue = new Queue(props.get("gcc-explorer", "maxConcurrentCompiles", 1), Infinity); +} + +Compile.prototype.newTempDir = function () { + return new Promise(function (resolve, reject) { + temp.mkdir('gcc-explorer-compiler', function (err, dirPath) { + if (err) + reject("Unable to open temp file: " + err); + else + resolve(dirPath); + }); + }); +}; + +Compile.prototype.writeFile = Promise.denodeify(fs.writeFile); +Compile.prototype.stat = Promise.denodeify(fs.stat); + +Compile.prototype.runCompiler = function(compiler, options) { + var okToCache = true; + var child = child_process.spawn( + compiler, + options, + {detached: true} + ); + var stdout = ""; + var stderr = ""; + var timeout = setTimeout(function () { + okToCache = false; + child.kill(); + stderr += "\nKilled - processing time exceeded"; + }, props.get("gcc-explorer", "compileTimeoutMs", 100)); + child.stdout.on('data', function (data) { + stdout += data; + }); + child.stderr.on('data', function (data) { + stderr += data; + }); + return new Promise(function (resolve) { + child.on('exit', function (code) { + clearTimeout(timeout); + resolve({code: code, stdout: stdout, stderr: stderr, okToCache: okToCache}); + }); + child.stdin.end(); + }); +}; + +Compile.prototype.compile = function (source, compiler, options, filters) { + var self = this; + return new Promise(function (resolve, reject) { + var optionsError = self.checkOptions(options); + if (optionsError) return reject(optionsError); + var sourceError = self.checkSource(source); + if (sourceError) return reject(sourceError); + + var compilerInfo = self.compilersByExe[compiler]; + if (!compilerInfo) { + return reject("Bad compiler " + compiler); + } + + var key = compiler + " | " + source + " | " + options + " | " + filters.intel; + var cached = self.cache.get(key); + if (cached) { + self.cacheHits++; + self.cacheStats(); + return resolve(cached); + } + self.cacheMisses++; + + var queuedPromise = self.compileQueue.add(function () { + return self.newTempDir().then(function (dirPath) { + var inputFilename = path.join(dirPath, props.get("gcc-explorer", "compileFilename")); + return self.writeFile(inputFilename, source).then(function () { + return {inputFilename: inputFilename, dirPath: dirPath}; + }); + }).then(function (info) { + var inputFilename = info.inputFilename; + var dirPath = info.dirPath; + var postProcess = props.get("gcc-explorer", "postProcess"); + var outputFilename = path.join(dirPath, 'output.S'); + if (compilerInfo.supportedOpts['-masm']) { + var syntax = '-masm=att'; // default at&t + if (filters.intel == "true") syntax = '-masm=intel'; + options = options.concat([syntax]); + } + var compileToAsm = props.get("gcc-explorer", "compileToAsm", "-S").split(" "); + options = options.concat(['-g', '-o', outputFilename]).concat(compileToAsm).concat([inputFilename]); + + var compilerWrapper = props.get("gcc-explorer", "compiler-wrapper"); + if (compilerWrapper) { + options = [compiler].concat(options); + compiler = compilerWrapper; + } + var maxSize = props.get("gcc-explorer", "max-asm-size", 8 * 1024 * 1024); + var gotAsmFinished = self.runCompiler(compiler, options).then(function (result) { + if (result.code !== 0) { + result.asm = ""; + return result; + } + return self.stat(outputFilename).then(function (stat) { + if (stat.size >= maxSize) { + result.asm = " " + maxSize + " bytes)>"; + return result; + } + return new Promise(function (resolve) { + child_process.exec('cat "' + outputFilename + '" | ' + postProcess, + {maxBuffer: maxSize}, + function (err, data) { + if (err) + data = ''; + result.asm = data; + resolve(result); + }); + }); + }, function () { + result.asm = ""; + return result; + }); + }); + + return gotAsmFinished.then(function (result) { + if (result.okToCache) { + self.cache.set(key, result); + self.cacheStats(); + } + return result; + }); + }); + }); + return queuedPromise.then(resolve, reject); + }); +}; + +Compile.prototype.checkOptions = function (options) { + var error = []; + var self = this; + options.forEach(function (option) { + if (!option.match(self.okOptions) || option.match(self.badOptions)) { + error.push(option); + } + }); + if (error.length > 0) return "Bad options: " + error.join(", "); + return null; +}; + +Compile.prototype.checkSource = function (source) { + var re = /^\s*#include(_next)?\s+["<"](\/|.*\.\.)/; + var failed = []; + source.split('\n').forEach(function (line, index) { + if (line.match(re)) { + failed.push(":" + (index + 1) + ":1: no absolute or relative includes please"); + } + }); + if (failed.length > 0) return failed.join("\n"); + return null; +}; + +Compile.prototype.cacheStats = function () { + console.log("Cache stats: " + this.cacheHits + " hits, " + this.cacheMisses + " misses"); +}; + +function compileHandler(compilers) { + var compileObj = new Compile(compilers); + return function compile(req, res) { + var source = req.body.source; + var compiler = req.body.compiler; + var options = req.body.options.split(' ').filter(function (x) { + return x !== ""; + }); + var filters = req.body.filters; + compileObj.compile(source, compiler, options, filters).then( + function (result) { + res.end(JSON.stringify(result)); + }, + function (error) { + if (typeof(error) !== "string") { + error = "Internal GCC explorer error: " + error.toString(); + } + res.end(JSON.stringify({code: -1, stderr: error})); + } + ); + }; +} + +module.exports = { + compileHandler: compileHandler +}; diff --git a/package.json b/package.json index 180e5aad..ae6672cc 100644 --- a/package.json +++ b/package.json @@ -16,13 +16,13 @@ }, "main": "./app.js", "dependencies": { - "async": "0.9.x", "body-parser": "1.10.x", "express": "4.11.x", "fs-extra": "0.8.x", "lru-cache": "2.5.x", "morgan": "1.5.x", "promise": "6.1.x", + "promise-queue": "2.1.x", "nopt": "3.0.x", "serve-favicon": "2.2.x", "serve-static": "1.8.x",