Browse Source

Fetch AWS resources (land of primitive service discovery)

dev/git-series/gccdum
Matt Godbolt 7 years ago
parent
commit
65e7678b64
  1. 100
      app.js
  2. 4
      etc/config/aws.lud-ldnmg01.properties
  3. 2
      etc/config/c++.lud-ldnmg01.properties
  4. 1
      etc/config/gcc-explorer.lud-ldnmg01.properties
  5. 27
      lib/aws.js
  6. 4
      package.json

100
app.js

@ -35,7 +35,9 @@ var nopt = require('nopt'),
http = require('http'),
https = require('https'),
url = require('url'),
Promise = require('promise');
Promise = require('promise'),
aws = require('./lib/aws'),
_ = require('underscore');
var opts = nopt({
'env': [String, Array],
@ -73,6 +75,13 @@ function compilerProps(property, defaultValue) {
require('./lib/compile').initialise(gccProps, compilerProps);
var staticMaxAgeMs = gccProps('staticMaxAgeMs', 0);
var awsProps = props.propsFor("aws");
var awsPoller = null;
function awsInstances() {
if (!awsPoller) awsPoller = new aws.InstanceFetcher(awsProps);
return awsPoller.getInstances();
}
function loadSources() {
var sourcesDir = "lib/sources";
var sources = fs.readdirSync(sourcesDir)
@ -206,40 +215,64 @@ function configuredCompilers() {
});
exes.push.apply(exes, toolchains);
}
// Map any named compilers to their executable
function fetchRemote(host, port) {
console.log("Fetching compilers from remote source " + host + ":" + port);
return retryPromise(function () {
return new Promise(function (resolve, reject) {
var request = http.get({
hostname: host,
port: port,
path: "/api/compilers"
}, function (res) {
var str = '';
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
var compilers = JSON.parse(str).map(function (compiler) {
compiler.exe = null;
compiler.remote = "http://" + host + ":" + port;
return compiler;
});
resolve(compilers);
});
}).on('error', function (e) {
reject(e);
}).on('timeout', function () {
reject("timeout");
});
request.setTimeout(1000);
});
},
host + ":" + port,
gccProps('proxyRetries', 20),
gccProps('proxyRetryMs', 500));
}
function fetchAws() {
console.log("Fetching instances from AWS");
return awsInstances().then(function (instances) {
return Promise.all(instances.map(function (instance) {
console.log("Checking instance " + instance.InstanceId);
var address = instance.PrivateDnsName;
if (awsProps("externalTestMode", false)) {
address = instance.PublicDnsName;
}
return fetchRemote(address, port);
}));
});
}
return Promise.all(exes.map(function (name) {
if (name.indexOf("@") !== -1) {
var bits = name.split("@");
var host = bits[0];
var port = parseInt(bits[1]);
console.log("Fetching compilers from remote source " + host + ":" + port);
return retryPromise(function () {
return new Promise(function (resolve, reject) {
http.get({
hostname: host,
port: port,
path: "/api/compilers"
}, function (res) {
var str = '';
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
var compilers = JSON.parse(str).map(function (compiler) {
compiler.exe = null;
compiler.remote = "http://" + host + ":" + port;
return compiler;
});
resolve(compilers);
});
}).on('error', function (e) {
reject(e);
});
});
},
host + ":" + port,
gccProps('proxyRetries', 20),
gccProps('proxyRetryMs', 500));
return fetchRemote(host, port);
}
if (name == "AWS") {
return fetchAws();
}
var base = "compiler." + name;
var exe = compilerProps(base + ".exe", "");
@ -263,11 +296,11 @@ function configuredCompilers() {
needsMulti: !!props("needsMulti", true),
supportsBinary: !!props("supportsBinary", true)
});
}));
})).then(_.flatten);
}
function getCompilerInfo(compilerInfo) {
if (Array.isArray(compilerInfo)) {
if (compilerInfo.remote) {
return Promise.resolve(compilerInfo);
}
return new Promise(function (resolve) {
@ -304,7 +337,6 @@ function findCompilers() {
return Promise.all(compilers.map(getCompilerInfo));
})
.then(function (compilers) {
compilers = Array.prototype.concat.apply([], compilers);
compilers = compilers.filter(function (x) {
return x !== null;
});
@ -399,7 +431,7 @@ findCompilers().then(function (compilers) {
var rescanCompilerSecs = gccProps('rescanCompilerSecs', 0);
if (rescanCompilerSecs) {
console.log("Rescanning compilers every " + rescanCompilerSecs + "secs");
setTimeout(function () {
setInterval(function () {
findCompilers().then(onCompilerChange);
}, rescanCompilerSecs * 1000);
}

4
etc/config/aws.lud-ldnmg01.properties

@ -0,0 +1,4 @@
region=us-east-1
tagKey=Name
tagValue=Sven
externalTestMode=true

2
etc/config/c++.lud-ldnmg01.properties

@ -1,6 +1,6 @@
# Default settings for GCC Explorer.
defaultCompiler=g52
compilers=g44:g45:g46:clang35:g51:g52:gdef:ec2-52-201-230-81.compute-1.amazonaws.com@10240
compilers=g44:g45:g46:clang35:g51:g52:gdef:AWS
#compilers=localhost@20480
compiler.g44.exe=/usr/bin/g++-4.4
compiler.g44.name=g++ 4.4

1
etc/config/gcc-explorer.lud-ldnmg01.properties

@ -1,2 +1,3 @@
language=C++
clientURLShortener=google
rescanCompilerSecs=360

27
lib/aws.js

@ -0,0 +1,27 @@
(function () {
function InstanceFetcher(properties) {
var self = this;
var AWS = require('aws-sdk');
var ec2 = new AWS.EC2({region: properties('region')});
var tagKey = properties('tagKey');
var tagValue = properties('tagValue');
this.onInstances = function (result) {
var allInstances = [];
result.Reservations.forEach(function (res) {
allInstances = allInstances.concat(res.Instances);
});
return allInstances.filter(function (reservation) {
return reservation.Tags.some(function (t) {
return t.Key == tagKey && t.Value == tagValue;
});
});
};
this.getInstances = function () {
return ec2.describeInstances().promise().then(self.onInstances);
};
}
exports.InstanceFetcher = InstanceFetcher;
}).call(this);

4
package.json

@ -16,6 +16,7 @@
},
"main": "./app.js",
"dependencies": {
"aws-sdk": "*",
"body-parser": "1.14.x",
"compression": "1.6.x",
"express": "4.13.x",
@ -28,7 +29,8 @@
"nopt": "3.0.x",
"serve-favicon": "2.3.x",
"serve-static": "1.10.x",
"temp": "0.8.x"
"temp": "0.8.x",
"underscore": "*"
},
"devDependencies": {
"supervisor": "0.3.1"

Loading…
Cancel
Save