Browse Source

Work in progress on refactoring to make it easier to support more compiler types

dev/git-series/gccdum
Matt Godbolt 6 years ago
parent
commit
b21e51ef27
  1. 211
      app.js
  2. 3
      etc/config/c++.amazon.properties
  3. 2
      etc/config/c++.lud-mgodbolt01.properties
  4. 4
      etc/config/c++.win32.properties
  5. 2
      etc/config/go.amazon.properties
  6. 2
      etc/config/go.lud-ldnmg01.properties
  7. 325
      lib/compile.js
  8. 3
      static/compiler.js

211
app.js

@ -133,6 +133,10 @@ fileSources.forEach(function (source) {
sourceToHandler[source.urlpart] = source;
});
var clientOptionsHandler = new ClientOptionsHandler(fileSources);
var apiHandler = new ApiHandler();
var compileHandler = new CompileHandler();
// auxiliary function used in clientOptionsHandler
function compareOn(key) {
return function (xObj, yObj) {
@ -174,6 +178,7 @@ function ClientOptionsHandler(fileSources) {
};
text = JSON.stringify(options);
};
this.setCompilers([]);
this.handler = function getClientOptions(req, res) {
res.set('Content-Type', 'application/json');
res.set('Cache-Control', 'public, max-age=' + staticMaxAgeSecs);
@ -232,9 +237,8 @@ function retryPromise(promiseFunc, name, maxFails, retryMs) {
});
}
// Auxiliary function to findCompilers()
function configuredCompilers() {
// read config (file already read) (':' are used to separate compilers names)
function findCompilers() {
var exes = compilerProps("compilers", "/usr/bin/g++").split(":");
var ndk = compilerProps('androidNdk');
if (ndk) {
@ -323,9 +327,7 @@ function configuredCompilers() {
options: props("options"),
versionFlag: props("versionFlag"),
versionRe: props("versionRe"),
is6g: !!props("is6g", false),
isCl: !!props("isCl", false),
needsWine: !!props("isCl", false) && process.platform == 'linux',
compilerType: props("compilerType", ""),
intelAsm: props("intelAsm", ""),
asmFlag: props("asmFlag", "-S"),
outputFlag: props("outputFlag", "-o"),
@ -361,75 +363,14 @@ function configuredCompilers() {
return Promise.resolve(compilerConfigFor(name, parentProps));
}
return Promise.all(exes.map(function (compiler) {
return recurseGetCompilers(compiler, compilerProps);
})).then(_.flatten);
}
// Auxiliary function to findCompilers()
function getCompilerInfo(compilerInfo) {
if (compilerInfo.remote) return Promise.resolve(compilerInfo);
return new Promise(function (resolve) {
var compiler = compilerInfo.exe;
var versionFlag = compilerInfo.versionFlag || '--version';
var versionRe = new RegExp(compilerInfo.versionRe || '.*');
var maybeWine = "";
if (compilerInfo.needsWine) {
maybeWine = '"' + gccProps("wine") + '" ';
}
logger.info("Gathering information on", compilerInfo);
// fill field compilerInfo.version,
// assuming the compiler returns its version on 1 line
child_process.exec(maybeWine + '"' + compiler + '" ' + versionFlag, function (err, output, stderr) {
if (err) {
logger.error("Unable to run compiler '" + compiler + "' : " + err);
return resolve(null);
}
var version = "";
_.each(utils.splitLines(output + stderr), function (line) {
if (version) return;
var match = line.match(versionRe);
if (match) version = match[0];
});
if (!version) {
logger.error("Unable to get compiler version for '" + compiler + "'");
return resolve(null);
}
logger.info(compiler + " is version '" + version + "'");
compilerInfo.version = version;
if (compilerInfo.intelAsm || compilerInfo.isCl) {
return resolve(compilerInfo);
}
// get information on the compiler's options
child_process.exec(compiler + ' --target-help', function (err, output) {
var options = {};
if (!err) {
var splitness = /--?[-a-zA-Z]+( ?[-a-zA-Z]+)/;
utils.eachLine(output, function (line) {
var match = line.match(splitness);
if (!match) return;
options[match[0]] = true;
});
}
if (options['-masm']) {
compilerInfo.intelAsm = "-masm=intel";
}
// debug (seems to be displayed multiple times):
logger.debug("compiler options: ", options);
resolve(compilerInfo);
});
});
});
}
function findCompilers() {
return configuredCompilers()
return Promise.all(
exes.map(function (compiler) {
return recurseGetCompilers(compiler, compilerProps);
})
)
.then(_.flatten)
.then(function (compilers) {
return Promise.all(compilers.map(getCompilerInfo));
return compileHandler.setCompilers(compilers)
})
.then(function (compilers) {
compilers = compilers.filter(function (x) {
@ -440,8 +381,8 @@ function findCompilers() {
});
}
// Instantiate a function that write informations on compiler,
// in JSON format (on which page ?)
// Instantiate a function that writes information on the compiler,
// in JSON format, for ease of external listing.
function ApiHandler() {
var reply = "";
this.setCompilers = function (compilers) {
@ -495,7 +436,8 @@ function shortUrlHandler(req, res, next) {
res.end();
});
}).on('error', function (e) {
res.end("TODO: error " + e.message);
logger.error("Error handling google URL shortener request", e);
res.end("Error " + e.message);
});
}
@ -508,71 +450,68 @@ function embeddedHandler(req, res, next) {
res.end();
}
var clientOptionsHandler = new ClientOptionsHandler(fileSources);
var apiHandler = new ApiHandler();
var compileHandler = new CompileHandler();
findCompilers().then(function (compilers) {
var prevCompilers;
findCompilers()
.then(function (compilers) {
var prevCompilers;
function onCompilerChange(compilers) {
if (JSON.stringify(prevCompilers) == JSON.stringify(compilers)) {
return;
}
logger.info("Compilers:", compilers);
if (compilers.length === 0) {
logger.error("#### No compilers found: no compilation will be done!");
function onCompilerChange(compilers) {
if (JSON.stringify(prevCompilers) == JSON.stringify(compilers)) {
return;
}
logger.info("Compilers:", compilers);
if (compilers.length === 0) {
logger.error("#### No compilers found: no compilation will be done!");
}
prevCompilers = compilers;
clientOptionsHandler.setCompilers(compilers);
apiHandler.setCompilers(compilers);
}
prevCompilers = compilers;
clientOptionsHandler.setCompilers(compilers);
apiHandler.setCompilers(compilers);
compileHandler.setCompilers(compilers);
}
onCompilerChange(compilers);
onCompilerChange(compilers);
var rescanCompilerSecs = gccProps('rescanCompilerSecs', 0);
if (rescanCompilerSecs) {
logger.info("Rescanning compilers every " + rescanCompilerSecs + "secs");
setInterval(function () {
findCompilers().then(onCompilerChange);
}, rescanCompilerSecs * 1000);
}
var rescanCompilerSecs = gccProps('rescanCompilerSecs', 0);
if (rescanCompilerSecs) {
logger.info("Rescanning compilers every " + rescanCompilerSecs + "secs");
setInterval(function () {
findCompilers().then(onCompilerChange);
}, rescanCompilerSecs * 1000);
}
var webServer = express(),
sFavicon = require('serve-favicon'),
sStatic = require('serve-static'),
bodyParser = require('body-parser'),
morgan = require('morgan'),
compression = require('compression'),
restreamer = require('./lib/restreamer'),
diffHandler = buildDiffHandler(wdiffConfig);
webServer
.set('trust proxy', true)
.use(morgan('combined', {stream: logger.stream}))
.use(compression())
.use(sFavicon(staticDir + '/favicon.ico'))
.use('/v', sStatic(staticDir + '/v', {maxAge: Infinity}))
.use(sStatic(staticDir, {maxAge: staticMaxAgeSecs * 1000}))
.use(bodyParser.json({limit: gccProps('bodyParserLimit', '1mb')}))
.use(restreamer())
.get('/client-options.json', clientOptionsHandler.handler)
.use('/source', getSource)
.use('/api', apiHandler.handler)
.use('/g', shortUrlHandler)
.use('/e', embeddedHandler)
.post('/compile', compileHandler.handler) // used inside static/compiler.js
.post('/diff', diffHandler); // used inside static/compiler.js
// GO!
logger.info("=======================================");
logger.info("Listening on http://" + hostname + ":" + port + "/");
logger.info(" serving static files from '" + staticDir + "'");
logger.info(" git release " + gitReleaseName);
logger.info("=======================================");
webServer.listen(port, hostname);
}).catch(function (err) {
var webServer = express(),
sFavicon = require('serve-favicon'),
sStatic = require('serve-static'),
bodyParser = require('body-parser'),
morgan = require('morgan'),
compression = require('compression'),
restreamer = require('./lib/restreamer'),
diffHandler = buildDiffHandler(wdiffConfig);
webServer
.set('trust proxy', true)
.use(morgan('combined', {stream: logger.stream}))
.use(compression())
.use(sFavicon(staticDir + '/favicon.ico'))
.use('/v', sStatic(staticDir + '/v', {maxAge: Infinity}))
.use(sStatic(staticDir, {maxAge: staticMaxAgeSecs * 1000}))
.use(bodyParser.json({limit: gccProps('bodyParserLimit', '1mb')}))
.use(restreamer())
.get('/client-options.json', clientOptionsHandler.handler)
.use('/source', getSource)
.use('/api', apiHandler.handler)
.use('/g', shortUrlHandler)
.use('/e', embeddedHandler)
.post('/compile', compileHandler.handler) // used inside static/compiler.js
.post('/diff', diffHandler); // used inside static/compiler.js
// GO!
logger.info("=======================================");
logger.info("Listening on http://" + hostname + ":" + port + "/");
logger.info(" serving static files from '" + staticDir + "'");
logger.info(" git release " + gitReleaseName);
logger.info("=======================================");
webServer.listen(port, hostname);
}).catch(function (err) {
logger.error("Error: " + err);
process.exit(1);
});

3
etc/config/c++.amazon.properties

@ -117,9 +117,8 @@ compiler.msp430g530.name=MSP430 gcc 5.3.0
################################
# Windows Compilers
group.cl.compilers=&cl19
group.cl.isCl=true
group.cl.needsMulti=false
group.cl.asmFlag=/FAsc
group.cl.compilerType=CL
group.cl.versionFlag=/?
group.cl.versionRe=^Microsoft \(R\) C/C\+\+.*$
group.cl19.compilers=cl19_64:cl19_32:cl19_arm

2
etc/config/c++.lud-mgodbolt01.properties

@ -16,7 +16,7 @@ compiler.clang39.exe=/usr/bin/clang++-3.9
compiler.clang39.name=Clang 3.9
group.windows.compilers=&cl19
group.windows.isCl=true
group.windows.compilerType=CL
group.windows.needsMulti=false
group.windows.asmFlag=/FAsc
group.windows.versionFlag=/?

4
etc/config/c++.win32.properties

@ -1,4 +1,2 @@
isCl=true
compileToAsm=/FAsc
compileToBinary=/FAsc
compilerType=CL
supportsBinary=true

2
etc/config/go.amazon.properties

@ -6,5 +6,5 @@ compiler.gccgo491.name=x86 gccgo 4.9.1
compiler.6g141.exe=/opt/gcc-explorer/go/pkg/tool/linux_amd64/6g
compiler.6g141.name=x86 6g 1.4.1
compiler.6g141.versionFlag=-V
compiler.6g141.is6g=true
compiler.6g141.compilerType=6g
compiler.6g141.supportsBinary=false

2
etc/config/go.lud-ldnmg01.properties

@ -4,4 +4,4 @@ compiler.go.name=GCC go
compiler.6g.exe=/home/mgodbolt/build/go/pkg/tool/linux_amd64/6g
compiler.6g.name=6g go
compiler.6g.versionFlag=-V
compiler.6g.is6g=true
compiler.6g.compilerType=6g

325
lib/compile.js

@ -83,29 +83,6 @@ Compile.prototype.writeFile = Promise.denodeify(fs.writeFile);
Compile.prototype.readFile = Promise.denodeify(fs.readFile);
Compile.prototype.stat = Promise.denodeify(fs.stat);
Compile.prototype.convert6g = function (code) {
var re = /^[0-9]+\s*\(([^:]+):([0-9]+)\)\s*([A-Z]+)(.*)/;
var prevLine = null;
var file = null;
return code.map(function (obj) {
var line = obj.line;
var match = line.match(re);
if (match) {
var res = "";
if (file === null) {
res += "\t.file 1 \"" + match[1] + "\"\n";
file = match[1];
}
if (prevLine != match[2]) {
res += "\t.loc 1 " + match[2] + "\n";
prevLine = match[2];
}
return res + "\t" + match[3].toLowerCase() + match[4];
} else
return null;
}).filter(identity).join("\n");
};
Compile.prototype.getRemote = function () {
if (this.compiler.exe === null && this.compiler.remote)
return this.compiler.remote;
@ -170,6 +147,10 @@ Compile.prototype.runCompiler = function (compiler, options, inputFilename) {
});
};
Compile.prototype.supportsObjdump = function () {
return true;
}
Compile.prototype.objdump = function (outputFilename, result, maxSize, intelAsm) {
var objDumpCommand = 'objdump -d -C "' + outputFilename + '" -l --insn-width=16';
if (intelAsm) objDumpCommand += " -M intel";
@ -185,6 +166,50 @@ Compile.prototype.objdump = function (outputFilename, result, maxSize, intelAsm)
});
};
Compile.prototype.wine = function () {
return "";
};
Compile.prototype.filename = function (fn) {
return fn;
};
Compile.prototype.optionsForFilter = function (filters, outputFilename) {
var options = ['-g', '-o', this.filename(outputFilename)];
if (this.compiler.intelAsm && filters.intel && !filters.binary) {
options = options.concat(this.compiler.intelAsm.split(" "));
}
if (!filters.binary) {
options = options.concat('-S');
}
// TODO: compileToAsm and compileToBinary compilerProps need folding into
// compiler types. info.asmFlag and info.outputFlag also
/*
var asmFlag = this.compiler.asmFlag ? this.compiler.asmFlag : "-S";
var outputFlag = this.compiler.outputFlag ? this.compiler.outputFlag : "-o";
var compileToAsm;
if (!filters.binary) {
compileToAsm = compilerProps("compileToAsm", asmFlag).split(" ");
} else {
compileToAsm = compilerProps("compileToBinary", "").split(" ");
}
*/
return options;
};
// TODO: where to draw the line between configuration and program behaviour?
// Per-compiler type stuff? probably things like asmFlag/outputFlag etc should be
// part of the compiler configuration in the compiler itself. intelAsm?
// options -- not options; this is reasonably per-compiler instance (e.g. cl include paths)
// clang gcc toolchain options? per compiler instance again?
Compile.prototype.prepareArguments = function (userOptions, filters, inputFilename, outputFilename) {
var options = this.optionsForFilter(filters, outputFilename);
if (this.compiler.options) {
options = options.concat(this.compiler.options.split(" "));
}
return options.concat(userOptions || []).concat([this.filename(inputFilename)]);
};
Compile.prototype.compile = function (source, options, filters) {
var self = this;
var optionsError = self.checkOptions(options);
@ -216,96 +241,32 @@ Compile.prototype.compile = function (source, options, filters) {
});
});
function filename(fn) {
if (self.compiler.needsWine) {
return 'Z:' + fn;
} else {
return fn;
}
}
var compileToAsmPromise = tempFileAndDirPromise.then(function (info) {
var inputFilename = info.inputFilename;
var dirPath = info.dirPath;
var postProcess = self.compiler.postProcess.filter(function (x) {
return x;
});
var outputFilename = path.join(dirPath, 'output.s'); // NB keep lower case as ldc compiler `tolower`s the output name
if (self.compiler.options) {
options = options.concat(self.compiler.options.split(" "));
}
if (self.compiler.intelAsm && filters.intel && !filters.binary) {
options = options.concat(self.compiler.intelAsm.split(" "));
}
var compileToAsm;
var asmFlag = self.compiler.asmFlag ? self.compiler.asmFlag : "-S";
var outputFlag = self.compiler.outputFlag ? self.compiler.outputFlag : "-o";
if (!filters.binary) {
compileToAsm = compilerProps("compileToAsm", asmFlag).split(" ");
} else {
compileToAsm = compilerProps("compileToBinary", "").split(" ");
}
if (self.compiler.isCl) {
options = options.concat(['/FAsc', '/c', '/Fa' + filename(outputFilename), '/Fo' + filename(outputFilename) + ".obj"]);
} else {
options = ['-g', outputFlag, filename(outputFilename)].concat(options);
}
options = options.concat(compileToAsm).concat([filename(inputFilename)]);
options = self.prepareArguments(options, filters, inputFilename, outputFilename);
var compilerExe = self.compiler.exe;
if (self.compiler.needsWine) {
if (self.wine()) {
options = [compilerExe].concat(options);
compilerExe = gccProps("wine");
compilerExe = self.wine();
}
var compilerWrapper = compilerProps("compiler-wrapper");
if (compilerWrapper) {
options = [compilerExe].concat(options);
compilerExe = compilerWrapper;
}
var maxSize = gccProps("max-asm-size", 8 * 1024 * 1024);
options = options.filter(identity);
return self.runCompiler(compilerExe, options, filename(inputFilename))
return self.runCompiler(compilerExe, options, self.filename(inputFilename))
.then(function (result) {
result.dirPath = dirPath;
if (result.code !== 0) {
result.asm = "<Compilation failed>";
return result;
}
if (self.compiler.is6g) {
result.asm = self.convert6g(result.stdout);
result.stdout = [];
return Promise.resolve(result);
}
if (filters.binary && !self.compiler.isCl) {
return self.objdump(outputFilename, result, maxSize, filters.intel);
}
return self.stat(outputFilename).then(function (stat) {
if (stat.size >= maxSize) {
result.asm = "<No output: generated assembly was too large (" + stat.size + " > " + maxSize + " bytes)>";
return result;
}
if (postProcess.length) {
return new Promise(function (resolve) {
var postCommand = 'cat "' + outputFilename + '" | ' + postProcess.join(" | ");
child_process.exec(postCommand,
{maxBuffer: maxSize},
function (err, data) {
if (err)
data = '<No output: ' + err + '>';
result.asm = data;
resolve(result);
});
});
} else {
return self.readFile(outputFilename).then(function (contents) {
result.asm = contents.toString();
return Promise.resolve(result);
});
}
}, function () {
result.asm = "<No output file>";
return result;
});
return self.postProcess(result, outputFilename, filters);
});
});
@ -326,6 +287,43 @@ Compile.prototype.compile = function (source, options, filters) {
});
};
Compile.prototype.postProcess = function (result, outputFilename, filters) {
var postProcess = this.compiler.postProcess.filter(identity);
var maxSize = gccProps("max-asm-size", 8 * 1024 * 1024);
if (filters.binary && this.supportsObjdump()) {
return this.objdump(outputFilename, result, maxSize, filters.intel);
}
return this.stat(outputFilename).then(_.bind(function (stat) {
if (stat.size >= maxSize) {
result.asm = "<No output: generated assembly was too large (" + stat.size + " > " + maxSize + " bytes)>";
return result;
}
if (postProcess.length) {
return new Promise(function (resolve) {
var postCommand = 'cat "' + outputFilename + '" | ' + postProcess.join(" | ");
child_process.exec(postCommand,
{maxBuffer: maxSize},
function (err, data) {
if (err)
data = '<No output: ' + err + '>';
result.asm = data;
resolve(result);
});
});
} else {
return this.readFile(outputFilename).then(function (contents) {
result.asm = contents.toString();
return Promise.resolve(result);
});
}
}, this),
function () {
result.asm = "<No output file>";
return result;
}
);
}
Compile.prototype.checkOptions = function (options) {
var error = this.env.findBadOptions(options);
if (error.length > 0) return "Bad options: " + error.join(", ");
@ -344,15 +342,150 @@ Compile.prototype.checkSource = function (source) {
return null;
};
Compile.prototype.initialise = function () {
if (this.getRemote()) return Promise.resolve(this);
return new Promise(_.bind(function (resolve) {
var compiler = this.compiler.exe;
var versionFlag = this.compiler.versionFlag || '--version';
var versionRe = new RegExp(this.compiler.versionRe || '.*');
var maybeWine = this.wine();
logger.info("Gathering version information on", compiler);
child_process.exec(maybeWine + ' "' + compiler + '" ' + versionFlag, _.bind(function (err, output, stderr) {
if (err) {
logger.error("Unable to run compiler '" + compiler + "' : " + err);
return resolve(null);
}
var version = "";
_.each(utils.splitLines(output + stderr), function (line) {
if (version) return;
var match = line.match(versionRe);
if (match) version = match[0];
});
if (!version) {
logger.error("Unable to get compiler version for '" + compiler + "'");
return resolve(null);
}
logger.info(compiler + " is version '" + version + "'");
this.compiler.version = version;
if (this.compiler.intelAsm) {
return resolve(this);
}
// get information on the compiler's options
child_process.exec(compiler + ' --target-help', _.bind(function (err, output) {
var options = {};
if (!err) {
var splitness = /--?[-a-zA-Z]+( ?[-a-zA-Z]+)/;
utils.eachLine(output, function (line) {
var match = line.match(splitness);
if (!match) return;
options[match[0]] = true;
});
}
if (options['-masm']) {
this.compiler.intelAsm = "-masm=intel";
}
// debug (seems to be displayed multiple times):
logger.debug("compiler options: ", options);
resolve(this);
}, this));
}, this));
}, this));
};
Compile.prototype.getInfo = function () {
return this.compiler;
};
function compileCl(info, env) {
var compile = new Compile(info, env);
info.supportsFiltersInBinary = true;
if (process.platform == "linux") {
var wine = gccProps("wine");
compile.wine = function () {
return wine;
};
compile.filename = function (fn) {
return 'Z:' + fn;
};
}
compile.supportsObjdump = function () {
return false;
};
compile.optionsForFilter = function (filters, outputFilename) {
return [
'/FAsc',
'/c',
'/Fa' + this.filename(outputFilename),
'/Fo' + this.filename(outputFilename + '.obj')
];
};
return compile.initialise();
}
function compile6g(info, env) {
function convert6g(code) {
var re = /^[0-9]+\s*\(([^:]+):([0-9]+)\)\s*([A-Z]+)(.*)/;
var prevLine = null;
var file = null;
return code.map(function (obj) {
var line = obj.line;
var match = line.match(re);
if (match) {
var res = "";
if (file === null) {
res += "\t.file 1 \"" + match[1] + "\"\n";
file = match[1];
}
if (prevLine != match[2]) {
res += "\t.loc 1 " + match[2] + "\n";
prevLine = match[2];
}
return res + "\t" + match[3].toLowerCase() + match[4];
} else
return null;
}).filter(identity).join("\n");
}
var compiler = new Compile(info, env);
compiler.postProcess = function () {
result.asm = this.convert6g(result.stdout);
result.stdout = [];
return Promise.resolve(result);
};
return compiler;
}
var compileFactories = {
"": function (info, env) {
var comp = new Compile(info, env);
return comp.initialise();
},
"CL": compileCl,
"6g": compile6g
};
function CompileHandler() {
this.compilersById = {};
this.compilerEnv = new CompilationEnvironment(gccProps);
this.setCompilers = function (compilers) {
this.compilersById = {};
_.each(compilers, function (compiler) {
this.compilersById[compiler.id] = new Compile(compiler, this.compilerEnv);
var initPromises = _.map(compilers, function (compiler) {
return compileFactories[compiler.compilerType](compiler, this.compilerEnv);
}, this);
return Promise.all(initPromises).then(_.bind(function (compilers) {
_.each(compilers, function (compiler) {
this.compilersById[compiler.compiler.id] = compiler;
}, this);
return _.map(compilers, function (compiler) {
return compiler.getInfo();
});
}, this)).catch(function (err) {
logger.error(err);
});
};
var proxy = httpProxy.createProxyServer({});

3
static/compiler.js

@ -352,7 +352,8 @@ define(function (require) {
this.domRoot.find("[data-bind='binary']")
.toggleClass("disabled", !this.compiler.supportsBinary);
// Disable any of the options which don't make sense in binary mode.
this.domRoot.find('.nonbinary').toggleClass("disabled", !!filters.binary && !this.compiler.isCl);
var filtersDisabled = !!filters.binary && !this.compiler.supportsFiltersInBinary;
this.domRoot.find('.nonbinary').toggleClass("disabled", filtersDisabled);
};
Compiler.prototype.onOptionsChange = function (options) {

Loading…
Cancel
Save