Browse Source

Misc WIP on changing configuration to be lang-specific and to fix up some more binary things found along the way

dev/git-series/gccdum
Matt Godbolt 7 years ago
parent
commit
79a0f6e12d
  1. 11
      Makefile
  2. 63
      app.js
  3. 9
      etc/config/c++.defaults.properties
  4. 18
      etc/config/c++.lud-ldnmg01.properties
  5. 3
      etc/config/d.defaults.properties
  6. 10
      etc/config/gcc-explorer.defaults.properties
  7. 40
      etc/config/gcc-explorer.lud-ldnmg01.properties
  8. 10
      etc/config/gcc-explorer.rust.properties
  9. 4
      etc/config/gcc-explorer.test1.properties
  10. 9
      etc/config/gcc-explorer.test2.properties
  11. 2
      etc/config/gcc-options.defaults.properties
  12. 4
      etc/config/go.defaults.properties
  13. 7
      etc/config/go.lud-ldnmg01.properties
  14. 9
      etc/config/rust.defaults.properties
  15. 14
      lib/asm.js
  16. 44
      lib/compile.js
  17. 14
      lib/properties.js
  18. 2
      lib/sources/builtin.js
  19. 3
      static/compiler.js

11
Makefile

@ -25,6 +25,8 @@ $(NODE_MODULES): package.json
npm install
@touch $@
LANG:=C++
node_modules: $(NODE_MODULES)
test:
@ -35,14 +37,7 @@ clean:
rm -rf node_modules .npm-updated
run: node_modules optional-d-support c-preload
$(NODE) ./node_modules/.bin/supervisor --exec $(NODE) ./app.js
$(NODE) ./node_modules/.bin/supervisor -w etc -e properties --exec $(NODE) -- ./app.js --language $(LANG)
c-preload:
$(MAKE) -C c-preload
run-amazon: node_modules optional-d-support c-preload
$(NODE) ./node_modules/.bin/supervisor --exec $(NODE) -- ./app.js --env amazon
run-amazon-d: node_modules optional-d-support c-preload
$(NODE) ./node_modules/.bin/supervisor --exec $(NODE) -- ./app.js --env amazon-d

63
app.js

@ -33,12 +33,13 @@ var nopt = require('nopt'),
path = require('path'),
fs = require('fs-extra'),
http = require('http'),
Promise = require('promise'),
heapdump = require('heapdump');
Promise = require('promise'); // TODO update npm list to remove heapdump
var opts = nopt({
'env': [String],
'rootDir': [String]
'rootDir': [String],
'language': [String],
'propDebug': [Boolean]
});
var propHierarchy = [
@ -47,11 +48,15 @@ var propHierarchy = [
os.hostname()];
var rootDir = opts.rootDir || './etc';
var language = opts.language || "C++";
props.initialize(rootDir + '/config', propHierarchy);
var port = props.get('gcc-explorer', 'port', 10240);
var staticMaxAgeMs = props.get('gcc-explorer', 'staticMaxAgeMs', 0);
if (opts.propDebug) props.setDebug(true);
var gccProps = props.propsFor("gcc-explorer");
var compilerProps = props.propsFor(language.toLowerCase());
require('./lib/compile').initialise(gccProps, compilerProps);
var port = gccProps('port', 10240);
var staticMaxAgeMs = gccProps('staticMaxAgeMs', 0);
function initializeMemwatch() {
var memwatch = require('memwatch-next');
@ -68,7 +73,7 @@ function initializeMemwatch() {
console.log("Memwatch stats: " + JSON.stringify(stats));
});
var heapDiffEverySecs = props.get('gcc-explorer', 'gcHeapDiffEverySecs', 0);
var heapDiffEverySecs = gccProps('gcHeapDiffEverySecs', 0);
if (heapDiffEverySecs) {
console.log("Diffing heap every " + heapDiffEverySecs + "s");
setInterval(function () {
@ -77,7 +82,7 @@ function initializeMemwatch() {
console.log("Memwatch diff from last stats: " + JSON.stringify(diff));
}, 1000 * heapDiffEverySecs);
}
var gcIntervalSecs = props.get("gcc-explorer", "gcIntervalSecs", 0);
var gcIntervalSecs = gccProps("gcIntervalSecs", 0);
if (gcIntervalSecs) {
console.log("Forcing a GC every " + gcIntervalSecs + "s");
setInterval(function () {
@ -121,17 +126,17 @@ function clientOptionsHandler(compilers, fileSources) {
});
sources = sources.sort(compareOn("name"));
var options = {
google_analytics_account: props.get('gcc-explorer', 'clientGoogleAnalyticsAccount', 'UA-55180-6'),
google_analytics_enabled: props.get('gcc-explorer', 'clientGoogleAnalyticsEnabled', false),
sharing_enabled: props.get('gcc-explorer', 'clientSharingEnabled', true),
github_ribbon_enabled: props.get('gcc-explorer', 'clientGitHubRibbonEnabled', true),
urlshortener: props.get('gcc-explorer', 'clientURLShortener', 'google'),
gapiKey: props.get('gcc-explorer', 'google-api-key', 'AIzaSyAaz35KJv8DA0ABoime0fEIh32NmbyYbcQ'),
defaultCompiler: props.get('gcc-explorer', 'defaultCompiler', ''),
defaultSource: props.get('gcc-explorer', 'defaultSource', ''),
google_analytics_account: gccProps('clientGoogleAnalyticsAccount', 'UA-55180-6'),
google_analytics_enabled: gccProps('clientGoogleAnalyticsEnabled', false),
sharing_enabled: gccProps('clientSharingEnabled', true),
github_ribbon_enabled: gccProps('clientGitHubRibbonEnabled', true),
urlshortener: gccProps('clientURLShortener', 'google'),
gapiKey: gccProps('google-api-key', 'AIzaSyAaz35KJv8DA0ABoime0fEIh32NmbyYbcQ'),
defaultSource: gccProps('defaultSource', ''),
language: language,
compilers: compilers,
language: props.get("gcc-explorer", "language"),
compileOptions: props.get("gcc-explorer", "options"),
defaultCompiler: compilerProps('defaultCompiler', ''),
compileOptions: compilerProps("options"),
sources: sources
};
var text = "var OPTIONS = " + JSON.stringify(options) + ";";
@ -193,8 +198,8 @@ function retryPromise(promiseFunc, name, maxFails, retryMs) {
}
function configuredCompilers() {
var exes = props.get("gcc-explorer", "compilers", "/usr/bin/g++").split(":");
var ndk = props.get('gcc-explorer', 'androidNdk');
var exes = compilerProps("compilers", "/usr/bin/g++").split(":");
var ndk = compilerProps('androidNdk');
if (ndk) {
var toolchains = fs.readdirSync(ndk + "/toolchains");
toolchains.forEach(function (v, i, a) {
@ -245,23 +250,23 @@ function configuredCompilers() {
});
},
host + ":" + port,
props.get('gcc-explorer', 'proxyRetries', 20),
props.get('gcc-explorer', 'proxyRetryMs', 500));
gccProps('proxyRetries', 20),
gccProps('proxyRetryMs', 500));
}
var base = "compiler." + name;
var exe = props.get("gcc-explorer", base + ".exe", "");
var exe = compilerProps(base + ".exe", "");
if (!exe) {
return Promise.resolve({id: name, exe: name, name: name});
}
return Promise.resolve({
id: name,
exe: exe,
name: props.get("gcc-explorer", base + ".name", name),
alias: props.get("gcc-explorer", base + ".alias"),
versionFlag: props.get("gcc-explorer", base + ".versionFlag"),
is6g: !!props.get("gcc-explorer", base + ".is6g", false),
intelAsm: props.get("gcc-explorer", base + ".intelAsm", ""),
supportsBinary: !!props.get("gcc-explorer", base + ".supportsBinary", false)
name: compilerProps(base + ".name", name),
alias: compilerProps(base + ".alias"),
versionFlag: compilerProps(base + ".versionFlag"),
is6g: !!compilerProps(base + ".is6g", false),
intelAsm: compilerProps(base + ".intelAsm", ""),
supportsBinary: !!compilerProps(base + ".supportsBinary", false)
});
}));
}

9
etc/config/c++.defaults.properties

@ -0,0 +1,9 @@
# Default settings for C++
compilers=/usr/bin/g++-4.4:/usr/bin/g++-4.5:/usr/bin/g++-4.6:/usr/bin/g++-4.7:/usr/bin/clang++:/usr/bin/g++
compileFilename=example.cpp
postProcess=c++filt
#androidNdk=/opt/google/android-ndk-r9c
options=-O2
binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy)$
stubRe=\bmain\b
stubText=int main(int,char**){/*stub provided by Compiler Explorer*/}

18
etc/config/c++.lud-ldnmg01.properties

@ -0,0 +1,18 @@
# Default settings for GCC Explorer.
defaultCompiler=g46
compilers=g44:g45:g46:clang35:g51
compiler.g44.exe=/usr/bin/g++-4.4
compiler.g44.name=g++ 4.4
compiler.g44.alias=/usr/bin/g++-4.4
compiler.g45.exe=/usr/bin/g++-4.5
compiler.g45.name=g++ 4.5
compiler.g45.alias=/usr/bin/g++-4.5
compiler.g46.exe=/usr/bin/g++-4.6
compiler.g46.name=g++ 4.6
compiler.g46.alias=/usr/bin/g++-4.6
compiler.clang35.exe=/usr/bin/clang++-3.5
compiler.clang35.name=Clang 3.5
compiler.clang35.intelAsm=-mllvm --x86-asm-syntax=intel
compiler.g51.exe=/home/mgodbolt/.fighome/runtime/gcc/5.1.0-1/bin/g++
compiler.g51.name=g++ 5.1
#compiler-wrapper=./c-preload/compiler-wrapper

3
etc/config/d.defaults.properties

@ -0,0 +1,3 @@
compilers=/usr/bin/gdc:/usr/bin/gdc-4.4:/usr/bin/gdc-4.6
compileFilename=example.d
postProcess=d/demangle

10
etc/config/gcc-explorer.defaults.properties

@ -1,18 +1,12 @@
# Default settings for GCC Explorer.
port=10240
compileTimeoutMs=1000
compilers=/usr/bin/g++-4.4:/usr/bin/g++-4.5:/usr/bin/g++-4.6:/usr/bin/g++-4.7:/usr/bin/clang++:/usr/bin/g++
defaultSource=builtin
compileFilename=example.cpp
postProcess=c++filt
cacheMb=50
#androidNdk=/opt/google/android-ndk-r9c
language=C++
options=-O2
maxConcurrentCompiles=2
staticMaxAgeMs=1000
gcIntervalSecs=300
gcHeapDiffEverySecs=3600
binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy)$
stubRe=\bmain\b
stubText=int main(int,char**){/*stub provided by Compiler Explorer*/}
optionsWhitelistRe=.*
optionsBlacklistRe=^((-wrapper|-fplugin?|(@.*)|-I|-i)(=.*)?|--)$

40
etc/config/gcc-explorer.lud-ldnmg01.properties

@ -1,39 +1 @@
# Default settings for GCC Explorer.
#port=10240
compileTimeoutMs=5000
defaultCompiler=g46
#compilers=g44:g45:g46:clang35:g51
compilers=go:6g
language=go
compileFilename=file.go
compiler.go.exe=/usr/bin/gccgo
compiler.go.name=GCC go
compiler.6g.exe=/home/mgodbolt/build/go/pkg/tool/linux_amd64/6g
compiler.6g.name=6g go
compiler.6g.versionFlag=-V
compiler.6g.is6g=true
compiler.g44.exe=/usr/bin/g++-4.4
compiler.g44.name=g++ 4.4
compiler.g44.alias=/usr/bin/g++-4.4
compiler.g45.exe=/usr/bin/g++-4.5
compiler.g45.name=g++ 4.5
compiler.g45.alias=/usr/bin/g++-4.5
compiler.g46.exe=/usr/bin/g++-4.6
compiler.g46.name=g++ 4.6
compiler.g46.alias=/usr/bin/g++-4.6
compiler.clang35.exe=/usr/bin/clang++-3.5
compiler.clang35.name=Clang 3.5
#compiler.clang35.intelAsm=-masm=intel
compiler.clang35.intelAsm=-mllvm --x86-asm-syntax=intel
compiler.g51.exe=/home/mgodbolt/.fighome/runtime/gcc/5.1.0-1/bin/g++
compiler.g51.name=g++ 5.1
#compilers=/usr/bin/g++-4.4:/usr/bin/g++-4.5:/usr/bin/g++-4.6:/usr/bin/g++-4.7:/usr/bin/clang++:/home/mgodbolt/apps/intel-icc-oss/bin/icc:/site/apps/gcc-4.7.2-drw.patched.6/bin/g++:/site/apps/gcc-4.8.0-drw.1/bin/g++
#compiler-wrapper=./c-preload/compiler-wrapper
#compilers=/usr/bin/gdc:/usr/bin/gdc-4.4:/usr/bin/gdc-4.6
#compileFilename=example.d
#postProcess=d/demangle
#TODO seriously consider moving all the lang-specific options into <lang>.properties
# for go:
binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy|.*@plt)$
stubRe=\bfunc\s+main\b
stubText=func main() {/*stub provided by Compiler Explorer*/}
language=C++

10
etc/config/gcc-explorer.rust.properties

@ -1,10 +0,0 @@
port=10242
compileTimeoutMs=1500
compilers=/usr/local/bin/rustc
compileFilename=example.rs
compileToAsm=--emit asm --crate-type staticlib
#compiler-wrapper=./c-preload/compiler-wrapper
#max-asm-size=262144
#cacheMb=100
language=Rust
options=-O --crate-type staticlib

4
etc/config/gcc-explorer.test1.properties

@ -1,4 +0,0 @@
compilers=localhost@20480:g44
compiler.g44.exe=/usr/bin/g++-4.4
compiler.g44.name=g++ 4.4
compiler.g44.alias=/usr/bin/g++-4.4

9
etc/config/gcc-explorer.test2.properties

@ -1,9 +0,0 @@
port=20480
compilers=g45:g46:clang34
compiler.g45.name=g++ 4.5
compiler.g45.alias=/usr/bin/g++-4.5
compiler.g46.exe=/usr/bin/g++-4.6
compiler.g46.name=g++ 4.6
compiler.g46.alias=/usr/bin/g++-4.6
compiler.clang34.exe=/usr/bin/clang++-3.4
compiler.clang34.name=Clang 4.6

2
etc/config/gcc-options.defaults.properties

@ -1,2 +0,0 @@
whitelistRe=.*
blacklistRe=^((-wrapper|-fplugin?|(@.*)|-I|-i)(=.*)?|--)$

4
etc/config/go.defaults.properties

@ -0,0 +1,4 @@
compileFilename=file.go
binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy|.*@plt)$
stubRe=\bfunc\s+main\b
stubText=func main() {/*stub provided by Compiler Explorer*/}

7
etc/config/go.lud-ldnmg01.properties

@ -0,0 +1,7 @@
compilers=go:6g
compiler.go.exe=/usr/bin/gccgo
compiler.go.name=GCC go
compiler.6g.exe=/home/mgodbolt/build/go/pkg/tool/linux_amd64/6g
compiler.6g.name=6g go
compiler.6g.versionFlag=-V
compiler.6g.is6g=true

9
etc/config/rust.defaults.properties

@ -0,0 +1,9 @@
compilers=/usr/local/bin/rustc
options=-O --crate-type staticlib
# TODO; remove need to have some kind of postprocess on everything
postProcess=c++filt
compileFilename=example.rs
compileToAsm=--emit asm --crate-type staticlib
stubRe=\bmain\b
stubText=pub fn main() {/*stub provided by Compiler Explorer*/}
binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy|.*@plt.*)$

14
lib/asm.js

@ -23,8 +23,6 @@
// POSSIBILITY OF SUCH DAMAGE.
(function () {
var props = require('./properties.js');
function processAsm(asm, filters) {
if (filters.binary) return processBinaryAsm(asm, filters);
@ -107,10 +105,12 @@
}
var binaryHideFuncRe = null;
function initialise() {
var pattern = props.get("gcc-explorer", 'binaryHideFuncRe');
var maxAsmLines = 500;
function initialise(compilerProps) {
var pattern = compilerProps('binaryHideFuncRe');
console.log("asm: binary re = " + pattern);
binaryHideFuncRe = new RegExp(pattern);
maxAsmLines = compilerProps('maxLinesOfAsm', maxAsmLines);
}
function isUserFunction(func) {
return !func.match(binaryHideFuncRe);
@ -126,6 +126,12 @@
var source = null;
var func = null;
asmLines.forEach(function (line) {
if (result.length >= maxAsmLines) {
if (result.length == maxAsmLines) {
result.push({text: "[truncated; too many lines]", source: null});
}
return;
}
var match = line.match(lineRe);
if (match) {
source = parseInt(match[2]);

44
lib/compile.js

@ -22,8 +22,7 @@
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
var props = require('./properties'),
child_process = require('child_process'),
var child_process = require('child_process'),
temp = require('temp'),
path = require('path'),
httpProxy = require('http-proxy'),
@ -42,17 +41,21 @@ function periodicCleanup() {
if (stats) console.log("Directory cleanup stats:", stats);
});
}
var gccProps = null;
var compilerProps = null;
var stubRe = null;
var stubText = null;
setTimeout(function () {
// property library isn't initialized until after the first tick
var tempDirCleanupSecs = props.get("gcc-explorer", "tempDirCleanupSecs", 600);
function initialise(gccProps_, compilerProps_) {
gccProps = gccProps_;
compilerProps = compilerProps_;
var tempDirCleanupSecs = gccProps("tempDirCleanupSecs", 600);
console.log("Cleaning temp dirs every " + tempDirCleanupSecs + " secs");
setInterval(periodicCleanup, tempDirCleanupSecs * 1000);
asm.initialise();
stubRe = props.get("gcc-explorer", "stubRe");
stubText = props.get("gcc-explorer", "stubText");
}, 0);
asm.initialise(compilerProps);
stubRe = compilerProps("stubRe");
stubText = compilerProps("stubText");
}
function Compile(compilers) {
this.compilersById = {};
@ -60,17 +63,17 @@ function Compile(compilers) {
compilers.forEach(function (compiler) {
self.compilersById[compiler.id] = compiler;
});
this.okOptions = new RegExp(props.get('gcc-options', 'whitelistRe', '.*'));
this.badOptions = new RegExp(props.get('gcc-options', 'blacklistRe'));
this.okOptions = new RegExp(gccProps('optionsWhitelistRe', '.*'));
this.badOptions = new RegExp(gccProps('optionsBlacklistRe'));
this.cache = LRU({
max: props.get('gcc-explorer', 'cacheMb') * 1024 * 1024,
max: gccProps('cacheMb') * 1024 * 1024,
length: function (n) {
return n.asm.length;
}
});
this.cacheHits = 0;
this.cacheMisses = 0;
this.compileQueue = new Queue(props.get("gcc-explorer", "maxConcurrentCompiles", 1), Infinity);
this.compileQueue = new Queue(gccProps("maxConcurrentCompiles", 1), Infinity);
}
Compile.prototype.newTempDir = function () {
@ -132,7 +135,7 @@ Compile.prototype.runCompiler = function (compiler, options) {
okToCache = false;
child.kill();
stderr += "\nKilled - processing time exceeded";
}, props.get("gcc-explorer", "compileTimeoutMs", 100));
}, gccProps("compileTimeoutMs", 100));
child.stdout.on('data', function (data) {
stdout += data;
});
@ -215,7 +218,7 @@ Compile.prototype.compile = function (source, compiler, options, filters) {
var tempFileAndDirPromise = Promise.resolve().then(function () {
return self.newTempDir().then(function (dirPath) {
var inputFilename = path.join(dirPath, props.get("gcc-explorer", "compileFilename"));
var inputFilename = path.join(dirPath, compilerProps("compileFilename"));
return self.writeFile(inputFilename, source).then(function () {
return {inputFilename: inputFilename, dirPath: dirPath};
});
@ -224,26 +227,26 @@ Compile.prototype.compile = function (source, compiler, options, filters) {
var compileToAsmPromise = tempFileAndDirPromise.then(function (info) {
var inputFilename = info.inputFilename;
var dirPath = info.dirPath;
var postProcess = props.get("gcc-explorer", "postProcess");
var postProcess = compilerProps("postProcess");
var outputFilename = path.join(dirPath, 'output.S');
if (compilerInfo.intelAsm && filters.intel && !filters.binary) {
options = options.concat(compilerInfo.intelAsm.split(" "));
}
var compileToAsm;
if (!filters.binary) {
compileToAsm = props.get("gcc-explorer", "compileToAsm", "-S").split(" ");
compileToAsm = compilerProps("compileToAsm", "-S").split(" ");
} else {
compileToAsm = [];
}
options = options.concat(['-g', '-o', outputFilename]).concat(compileToAsm).concat([inputFilename]);
var compilerExe = compilerInfo.exe;
var compilerWrapper = props.get("gcc-explorer", "compiler-wrapper");
var compilerWrapper = compilerProps("compiler-wrapper");
if (compilerWrapper) {
options = [compilerExe].concat(options);
compilerExe = compilerWrapper;
}
var maxSize = props.get("gcc-explorer", "max-asm-size", 8 * 1024 * 1024);
var maxSize = gccProps("max-asm-size", 8 * 1024 * 1024);
return self.runCompiler(compilerExe, options).then(function (result) {
result.dirPath = dirPath;
if (result.code !== 0) {
@ -364,5 +367,6 @@ function compileHandler(compilers) {
}
module.exports = {
compileHandler: compileHandler
compileHandler: compileHandler,
initialise: initialise
};

14
lib/properties.js

@ -28,13 +28,15 @@ var properties = {};
var hierarchy = [];
var propDebug = false;
function findProps(base, elem) {
var name = base + '.' + elem;
return properties[name];
}
function debug(string) {
//console.log("prop: " + string);
if (propDebug) console.log("prop: " + string);
}
function get(base, property, defaultValue) {
@ -93,7 +95,15 @@ function initialize(directory, hier) {
});
}
function propsFor(base) {
return function(property, defaultValue) {
return get(base, property, defaultValue);
}
}
module.exports = {
get: get,
initialize: initialize
propsFor: propsFor,
initialize: initialize,
setDebug: function(debug) { propDebug = debug; }
};

2
lib/sources/builtin.js

@ -27,7 +27,7 @@
var props = require('../properties.js'),
path = require('path'),
fs = require('fs');
//TODO update this to use compiler options?
var sourcePath = props.get('builtin', 'sourcepath', './examples/c++');
var sourceMatch = new RegExp(props.get('builtin', 'extensionRe', '.*\\.cpp$'));
var examples = fs.readdirSync(sourcePath)

3
static/compiler.js

@ -343,7 +343,8 @@ function Compiler(domRoot, origFilters, windowLocalPrefix, onChangeCallback, lan
var compiler = compilersById[$('.compiler').val()];
if (compiler === undefined)
return;
domRoot.find('.filter button.btn[value="intel"]').toggleClass("disabled", !compiler.intelAsm);
var supportsIntel = compiler.asm || filters.binary; // TODO: separate binary so it has its own buttons and then this problem goes away
domRoot.find('.filter button.btn[value="intel"]').toggleClass("disabled", !supportsIntel);
$(".compilerVersion").text(compiler.name + " (" + compiler.version + ")");
}

Loading…
Cancel
Save