Browse Source

First attempt at proxying remote compilers

dev/git-series/gccdum
Matt Godbolt 8 years ago
parent
commit
2e88d332b4
  1. 145
      app.js
  2. 2
      etc/config/gcc-explorer.lud-ldnmg01.properties
  3. 4
      etc/config/gcc-explorer.test1.properties
  4. 9
      etc/config/gcc-explorer.test2.properties
  5. 21
      lib/compile.js
  6. 2
      package.json
  7. 3
      static/compiler.js

145
app.js

@ -32,6 +32,7 @@ var nopt = require('nopt'),
child_process = require('child_process'),
path = require('path'),
fs = require('fs-extra'),
http = require('http'),
Promise = require('promise');
var opts = nopt({
@ -79,6 +80,32 @@ function compareOn(key) {
};
}
function clientOptionsHandler(compilers, fileSources) {
var sources = fileSources.map(function (source) {
return {name: source.name, urlpart: source.urlpart};
});
sources = sources.sort(compareOn("name"));
var options = {
google_analytics_account: props.get('gcc-explorer', 'clientGoogleAnalyticsAccount', 'UA-55180-6'),
google_analytics_enabled: props.get('gcc-explorer', 'clientGoogleAnalyticsEnabled', false),
sharing_enabled: props.get('gcc-explorer', 'clientSharingEnabled', true),
github_ribbon_enabled: props.get('gcc-explorer', 'clientGitHubRibbonEnabled', true),
urlshortener: props.get('gcc-explorer', 'clientURLShortener', 'google'),
defaultCompiler: props.get('gcc-explorer', 'defaultCompiler', ''),
defaultSource: props.get('gcc-explorer', 'defaultSource', ''),
compilers: compilers,
language: props.get("gcc-explorer", "language"),
compileOptions: props.get("gcc-explorer", "options"),
sources: sources
};
var text = "var OPTIONS = " + JSON.stringify(options) + ";";
return function getClientOptions(req, res) {
res.set('Content-Type', 'application/javascript');
res.set('Cache-Control', 'public, max-age=' + staticMaxAgeMs);
res.end(text);
};
}
function getSource(req, res, next) {
var bits = req.url.split("/");
var handler = sourceToHandler[bits[1]];
@ -127,48 +154,53 @@ function configuredCompilers() {
exes.push.apply(exes, toolchains);
}
// Map any named compilers to their executable
return exes.map(function (name) {
return Promise.all(exes.map(function (name) {
if (name.indexOf("@") !== -1) {
var bits = name.split("@");
var host = bits[0];
var port = parseInt(bits[1]);
console.log("Fetching compilers from remote source " + host + ":" + port);
return new Promise(function (resolve, reject) {
http.get({
hostname: host,
port: port,
path: "/api/compilers"
}, function (res) {
var str = '';
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
var compilers = JSON.parse(str).map(function (compiler) {
compiler.exe = null;
compiler.remote = "http://" + host + ":" + port;
return compiler;
});
resolve(compilers);
});
}).on('error', function (e) {
reject(e);
});
});
}
var base = "compiler." + name;
var exe = props.get("gcc-explorer", base + ".exe", "");
if (!exe) {
return {id: name, exe: name, name: name};
return Promise.resolve({id: name, exe: name, name: name});
}
return {
return Promise.resolve({
id: name,
exe: exe,
name: props.get("gcc-explorer", base + ".name", name),
alias: props.get("gcc-explorer", base + ".alias")
};
});
}
function clientOptionsHandler(compilers, fileSources) {
var sources = fileSources.map(function (source) {
return {name: source.name, urlpart: source.urlpart};
});
sources = sources.sort(compareOn("name"));
var options = {
google_analytics_account: props.get('gcc-explorer', 'clientGoogleAnalyticsAccount', 'UA-55180-6'),
google_analytics_enabled: props.get('gcc-explorer', 'clientGoogleAnalyticsEnabled', false),
sharing_enabled: props.get('gcc-explorer', 'clientSharingEnabled', true),
github_ribbon_enabled: props.get('gcc-explorer', 'clientGitHubRibbonEnabled', true),
urlshortener: props.get('gcc-explorer', 'clientURLShortener', 'google'),
defaultCompiler: props.get('gcc-explorer', 'defaultCompiler', ''),
defaultSource: props.get('gcc-explorer', 'defaultSource', ''),
compilers: compilers,
language: props.get("gcc-explorer", "language"),
compileOptions: props.get("gcc-explorer", "options"),
sources: sources
};
var text = "var OPTIONS = " + JSON.stringify(options) + ";";
return function getClientOptions(req, res) {
res.set('Content-Type', 'application/javascript');
res.set('Cache-Control', 'public, max-age=' + staticMaxAgeMs);
res.end(text);
};
});
}));
}
function getCompilerInfo(compilerInfo) {
if (Array.isArray(compilerInfo)) {
return Promise.resolve(compilerInfo);
}
return new Promise(function (resolve) {
var compiler = compilerInfo.exe;
child_process.exec(compiler + ' --version', function (err, output) {
@ -193,16 +225,42 @@ function getCompilerInfo(compilerInfo) {
}
function findCompilers() {
var compilers = configuredCompilers().map(getCompilerInfo);
return Promise.all(compilers).then(function (compilers) {
compilers = compilers.filter(function (x) {
return x !== null;
});
compilers = compilers.sort(function (x, y) {
return x.name < y.name ? -1 : x.name > y.name ? 1 : 0;
return configuredCompilers()
.then(function (compilers) {
return Promise.all(compilers.map(getCompilerInfo));
})
.then(function (compilers) {
compilers = Array.prototype.concat.apply([], compilers);
compilers = compilers.filter(function (x) {
return x !== null;
});
compilers = compilers.sort(function (x, y) {
return x.name < y.name ? -1 : x.name > y.name ? 1 : 0;
});
console.log("Compilers:");
compilers.forEach(function (c) {
console.log(c.id + " : " + c.name + " : " + (c.exe || c.remote));
});
return compilers;
});
return compilers;
});
}
function apiHandler(compilers) {
var reply = JSON.stringify(compilers);
return function apiHandler(req, res, next) {
var bits = req.url.split("/");
if (bits.length !== 2 || req.method !== "GET") return next();
switch (bits[1]) {
default:
next();
break;
case "compilers":
res.set('Content-Type', 'application/json');
res.end(reply);
break;
}
};
}
findCompilers().then(function (compilers) {
@ -211,16 +269,19 @@ findCompilers().then(function (compilers) {
sStatic = require('serve-static'),
bodyParser = require('body-parser'),
logger = require('morgan'),
compression = require('compression');
compression = require('compression'),
restreamer = require('connect-restreamer');
webServer
.use(logger('combined'))
.use(compression())
.use(sFavicon('static/favicon.ico'))
.use(sStatic('static', {maxAge: staticMaxAgeMs}))
.use(bodyParser.urlencoded({extended: true}))
.use(bodyParser.json())
.use(restreamer())
.get('/client-options.js', clientOptionsHandler(compilers, fileSources))
.use('/source', getSource)
.use('/api', apiHandler(compilers))
.post('/compile', compileHandler(compilers));
// GO!

2
etc/config/gcc-explorer.lud-ldnmg01.properties

@ -2,7 +2,7 @@
#port=10240
compileTimeoutMs=5000
defaultCompiler=g46
compilers=g44:g45:g46:clang34
#compilers=g44:g45:g46:clang34
compiler.g44.exe=/usr/bin/g++-4.4
compiler.g44.name=g++ 4.4
compiler.g44.alias=/usr/bin/g++-4.4

4
etc/config/gcc-explorer.test1.properties

@ -0,0 +1,4 @@
compilers=localhost@20480:g44
compiler.g44.exe=/usr/bin/g++-4.4
compiler.g44.name=g++ 4.4
compiler.g44.alias=/usr/bin/g++-4.4

9
etc/config/gcc-explorer.test2.properties

@ -0,0 +1,9 @@
port=20480
compilers=g45:g46:clang34
compiler.g45.name=g++ 4.5
compiler.g45.alias=/usr/bin/g++-4.5
compiler.g46.exe=/usr/bin/g++-4.6
compiler.g46.name=g++ 4.6
compiler.g46.alias=/usr/bin/g++-4.6
compiler.clang34.exe=/usr/bin/clang++-3.4
compiler.clang34.name=Clang 4.6

21
lib/compile.js

@ -26,6 +26,7 @@ var props = require('./properties'),
child_process = require('child_process'),
temp = require('temp'),
path = require('path'),
httpProxy = require('http-proxy'),
LRU = require('lru-cache'),
fs = require('fs-extra'),
Promise = require('promise'),
@ -67,6 +68,14 @@ Compile.prototype.newTempDir = function () {
Compile.prototype.writeFile = Promise.denodeify(fs.writeFile);
Compile.prototype.stat = Promise.denodeify(fs.stat);
Compile.prototype.getRemote = function (compiler) {
var compilerInfo = this.compilersById[compiler];
if (!compilerInfo) return false;
if (compilerInfo.exe === null && compilerInfo.remote)
return compilerInfo.remote;
return false;
};
Compile.prototype.runCompiler = function (compiler, options) {
var okToCache = true;
var child = child_process.spawn(
@ -221,9 +230,19 @@ Compile.prototype.cacheStats = function () {
function compileHandler(compilers) {
var compileObj = new Compile(compilers);
var proxy = httpProxy.createProxyServer({});
return function compile(req, res) {
var source = req.body.source;
var compiler = req.body.compiler;
var remote = compileObj.getRemote(compiler);
if (remote) {
proxy.web(req, res, {target:remote}, function (e) {
console.log("Proxy error: ",e);
res.end();
});
return;
}
var source = req.body.source;
var options = req.body.options.split(' ').filter(function (x) {
return x !== "";
});

2
package.json

@ -18,8 +18,10 @@
"dependencies": {
"body-parser": "1.10.x",
"compression": "1.3.x",
"connect-restreamer": "1.0.x",
"express": "4.11.x",
"fs-extra": "0.8.x",
"http-proxy": "1.8.x",
"lru-cache": "2.5.x",
"morgan": "1.5.x",
"promise": "6.1.x",

3
static/compiler.js

@ -214,7 +214,8 @@ function Compiler(domRoot, origFilters, windowLocalPrefix, onChangeCallback, cmM
type: 'POST',
url: '/compile',
dataType: 'json',
data: data,
contentType: 'application/json',
data: JSON.stringify(data),
success: function (result) {
onCompileResponse(data, result);
}

Loading…
Cancel
Save