Fetch AWS resources (land of primitive service discovery)

app.js

@@ -35,7 +35,9 @@ var nopt = require('nopt'),
     http = require('http'),
     https = require('https'),
     url = require('url'),
-    Promise = require('promise');
+    Promise = require('promise'),
+    aws = require('./lib/aws'),
+    _ = require('underscore');
 
 var opts = nopt({
     'env': [String, Array],
@@ -73,6 +75,13 @@ function compilerProps(property, defaultValue) {
 require('./lib/compile').initialise(gccProps, compilerProps);
 var staticMaxAgeMs = gccProps('staticMaxAgeMs', 0);
 
+var awsProps = props.propsFor("aws");
+var awsPoller = null;
+function awsInstances() {
+    if (!awsPoller) awsPoller = new aws.InstanceFetcher(awsProps);
+    return awsPoller.getInstances();
+}
+
 function loadSources() {
     var sourcesDir = "lib/sources";
     var sources = fs.readdirSync(sourcesDir)
@@ -206,40 +215,64 @@ function configuredCompilers() {
         });
         exes.push.apply(exes, toolchains);
     }
-    // Map any named compilers to their executable
+
+    function fetchRemote(host, port) {
+        console.log("Fetching compilers from remote source " + host + ":" + port);
+        return retryPromise(function () {
+                return new Promise(function (resolve, reject) {
+                    var request = http.get({
+                        hostname: host,
+                        port: port,
+                        path: "/api/compilers"
+                    }, function (res) {
+                        var str = '';
+                        res.on('data', function (chunk) {
+                            str += chunk;
+                        });
+                        res.on('end', function () {
+                            var compilers = JSON.parse(str).map(function (compiler) {
+                                compiler.exe = null;
+                                compiler.remote = "http://" + host + ":" + port;
+                                return compiler;
+                            });
+                            resolve(compilers);
+                        });
+                    }).on('error', function (e) {
+                        reject(e);
+                    }).on('timeout', function () {
+                        reject("timeout");
+                    });
+                    request.setTimeout(1000);
+                });
+            },
+            host + ":" + port,
+            gccProps('proxyRetries', 20),
+            gccProps('proxyRetryMs', 500));
+    }
+
+    function fetchAws() {
+        console.log("Fetching instances from AWS");
+        return awsInstances().then(function (instances) {
+            return Promise.all(instances.map(function (instance) {
+                console.log("Checking instance " + instance.InstanceId);
+                var address = instance.PrivateDnsName;
+                if (awsProps("externalTestMode", false)) {
+                    address = instance.PublicDnsName;
+                }
+                return fetchRemote(address, port);
+            }));
+        });
+    }
+
     return Promise.all(exes.map(function (name) {
         if (name.indexOf("@") !== -1) {
             var bits = name.split("@");
             var host = bits[0];
             var port = parseInt(bits[1]);
-            console.log("Fetching compilers from remote source " + host + ":" + port);
-            return retryPromise(function () {
-                    return new Promise(function (resolve, reject) {
-                        http.get({
-                            hostname: host,
-                            port: port,
-                            path: "/api/compilers"
-                        }, function (res) {
-                            var str = '';
-                            res.on('data', function (chunk) {
-                                str += chunk;
-                            });
-                            res.on('end', function () {
-                                var compilers = JSON.parse(str).map(function (compiler) {
-                                    compiler.exe = null;
-                                    compiler.remote = "http://" + host + ":" + port;
-                                    return compiler;
-                                });
-                                resolve(compilers);
-                            });
-                        }).on('error', function (e) {
-                            reject(e);
-                        });
-                    });
-                },
-                host + ":" + port,
-                gccProps('proxyRetries', 20),
-                gccProps('proxyRetryMs', 500));
+            return fetchRemote(host, port);
+        }
+        if (name == "AWS") {
+            return fetchAws();
         }
         var base = "compiler." + name;
         var exe = compilerProps(base + ".exe", "");
@@ -263,11 +296,11 @@ function configuredCompilers() {
             needsMulti: !!props("needsMulti", true),
             supportsBinary: !!props("supportsBinary", true)
         });
-    }));
+    })).then(_.flatten);
 }
 
 function getCompilerInfo(compilerInfo) {
-    if (Array.isArray(compilerInfo)) {
+    if (compilerInfo.remote) {
         return Promise.resolve(compilerInfo);
     }
     return new Promise(function (resolve) {
@@ -304,7 +337,6 @@ function findCompilers() {
             return Promise.all(compilers.map(getCompilerInfo));
         })
         .then(function (compilers) {
-            compilers = Array.prototype.concat.apply([], compilers);
             compilers = compilers.filter(function (x) {
                 return x !== null;
             });
@@ -399,7 +431,7 @@ findCompilers().then(function (compilers) {
     var rescanCompilerSecs = gccProps('rescanCompilerSecs', 0);
     if (rescanCompilerSecs) {
         console.log("Rescanning compilers every " + rescanCompilerSecs + "secs");
-        setTimeout(function () {
+        setInterval(function () {
             findCompilers().then(onCompilerChange);
         }, rescanCompilerSecs * 1000);
     }

etc/config/aws.lud-ldnmg01.properties

@@ -0,0 +1,4 @@
+region=us-east-1
+tagKey=Name
+tagValue=Sven
+externalTestMode=true

etc/config/c++.lud-ldnmg01.properties

@@ -1,6 +1,6 @@
 # Default settings for GCC Explorer.
 defaultCompiler=g52
-compilers=g44:g45:g46:clang35:g51:g52:gdef:ec2-52-201-230-81.compute-1.amazonaws.com@10240
+compilers=g44:g45:g46:clang35:g51:g52:gdef:AWS
 #compilers=localhost@20480
 compiler.g44.exe=/usr/bin/g++-4.4
 compiler.g44.name=g++ 4.4

etc/config/gcc-explorer.lud-ldnmg01.properties

@@ -1,2 +1,3 @@
 language=C++
 clientURLShortener=google
+rescanCompilerSecs=360

lib/aws.js

@@ -0,0 +1,27 @@
+(function () {
+    function InstanceFetcher(properties) {
+        var self = this;
+        var AWS = require('aws-sdk');
+        var ec2 = new AWS.EC2({region: properties('region')});
+        var tagKey = properties('tagKey');
+        var tagValue = properties('tagValue');
+
+        this.onInstances = function (result) {
+            var allInstances = [];
+            result.Reservations.forEach(function (res) {
+                allInstances = allInstances.concat(res.Instances);
+            });
+            return allInstances.filter(function (reservation) {
+                return reservation.Tags.some(function (t) {
+                    return t.Key == tagKey && t.Value == tagValue;
+                });
+            });
+        };
+
+        this.getInstances = function () {
+            return ec2.describeInstances().promise().then(self.onInstances);
+        };
+    }
+
+    exports.InstanceFetcher = InstanceFetcher;
+}).call(this);

package.json

@@ -16,6 +16,7 @@
   },
   "main": "./app.js",
   "dependencies": {
+    "aws-sdk": "*",
     "body-parser": "1.14.x",
     "compression": "1.6.x",
     "express": "4.13.x",
@@ -28,7 +29,8 @@
     "nopt": "3.0.x",
     "serve-favicon": "2.3.x",
     "serve-static": "1.10.x",
-    "temp": "0.8.x"
+    "temp": "0.8.x",
+    "underscore": "*"
   },
   "devDependencies": {
     "supervisor": "0.3.1"