Misc WIP on changing configuration to be lang-specific and to fix up some more binary things found along the way

Makefile

@@ -25,6 +25,8 @@ $(NODE_MODULES): package.json
 	npm install
 	@touch $@
 
+LANG:=C++
+
 node_modules: $(NODE_MODULES)
 
 test:
@@ -35,14 +37,7 @@ clean:
 	rm -rf node_modules .npm-updated
 
 run: node_modules optional-d-support c-preload
-	$(NODE) ./node_modules/.bin/supervisor --exec $(NODE) ./app.js
+	$(NODE) ./node_modules/.bin/supervisor -w etc -e properties --exec $(NODE) -- ./app.js --language $(LANG)
 
 c-preload:
 	$(MAKE) -C c-preload
-
-run-amazon: node_modules optional-d-support c-preload
-	$(NODE) ./node_modules/.bin/supervisor --exec $(NODE) -- ./app.js --env amazon
-
-run-amazon-d: node_modules optional-d-support c-preload
-	$(NODE) ./node_modules/.bin/supervisor --exec $(NODE) -- ./app.js --env amazon-d
-

app.js

@@ -33,12 +33,13 @@ var nopt = require('nopt'),
     path = require('path'),
     fs = require('fs-extra'),
     http = require('http'),
-    Promise = require('promise'),
-    heapdump = require('heapdump');
+    Promise = require('promise');  // TODO update npm list to remove heapdump
 
 var opts = nopt({
     'env': [String],
-    'rootDir': [String]
+    'rootDir': [String],
+    'language': [String],
+    'propDebug': [Boolean]
 });
 
 var propHierarchy = [
@@ -47,11 +48,15 @@ var propHierarchy = [
     os.hostname()];
 
 var rootDir = opts.rootDir || './etc';
+var language = opts.language || "C++";
 
 props.initialize(rootDir + '/config', propHierarchy);
-
-var port = props.get('gcc-explorer', 'port', 10240);
-var staticMaxAgeMs = props.get('gcc-explorer', 'staticMaxAgeMs', 0);
+if (opts.propDebug) props.setDebug(true);
+var gccProps = props.propsFor("gcc-explorer");
+var compilerProps = props.propsFor(language.toLowerCase());
+require('./lib/compile').initialise(gccProps, compilerProps);
+var port = gccProps('port', 10240);
+var staticMaxAgeMs = gccProps('staticMaxAgeMs', 0);
 
 function initializeMemwatch() {
     var memwatch = require('memwatch-next');
@@ -68,7 +73,7 @@ function initializeMemwatch() {
             console.log("Memwatch stats: " + JSON.stringify(stats));
         });
 
-        var heapDiffEverySecs = props.get('gcc-explorer', 'gcHeapDiffEverySecs', 0);
+        var heapDiffEverySecs = gccProps('gcHeapDiffEverySecs', 0);
         if (heapDiffEverySecs) {
             console.log("Diffing heap every " + heapDiffEverySecs + "s");
             setInterval(function () {
@@ -77,7 +82,7 @@ function initializeMemwatch() {
                 console.log("Memwatch diff from last stats: " + JSON.stringify(diff));
             }, 1000 * heapDiffEverySecs);
         }
-        var gcIntervalSecs = props.get("gcc-explorer", "gcIntervalSecs", 0);
+        var gcIntervalSecs = gccProps("gcIntervalSecs", 0);
         if (gcIntervalSecs) {
             console.log("Forcing a GC every " + gcIntervalSecs + "s");
             setInterval(function () {
@@ -121,17 +126,17 @@ function clientOptionsHandler(compilers, fileSources) {
     });
     sources = sources.sort(compareOn("name"));
     var options = {
-        google_analytics_account: props.get('gcc-explorer', 'clientGoogleAnalyticsAccount', 'UA-55180-6'),
-        google_analytics_enabled: props.get('gcc-explorer', 'clientGoogleAnalyticsEnabled', false),
-        sharing_enabled: props.get('gcc-explorer', 'clientSharingEnabled', true),
-        github_ribbon_enabled: props.get('gcc-explorer', 'clientGitHubRibbonEnabled', true),
-        urlshortener: props.get('gcc-explorer', 'clientURLShortener', 'google'),
-        gapiKey: props.get('gcc-explorer', 'google-api-key', 'AIzaSyAaz35KJv8DA0ABoime0fEIh32NmbyYbcQ'),
-        defaultCompiler: props.get('gcc-explorer', 'defaultCompiler', ''),
-        defaultSource: props.get('gcc-explorer', 'defaultSource', ''),
+        google_analytics_account: gccProps('clientGoogleAnalyticsAccount', 'UA-55180-6'),
+        google_analytics_enabled: gccProps('clientGoogleAnalyticsEnabled', false),
+        sharing_enabled: gccProps('clientSharingEnabled', true),
+        github_ribbon_enabled: gccProps('clientGitHubRibbonEnabled', true),
+        urlshortener: gccProps('clientURLShortener', 'google'),
+        gapiKey: gccProps('google-api-key', 'AIzaSyAaz35KJv8DA0ABoime0fEIh32NmbyYbcQ'),
+        defaultSource: gccProps('defaultSource', ''),
+        language: language,
         compilers: compilers,
-        language: props.get("gcc-explorer", "language"),
-        compileOptions: props.get("gcc-explorer", "options"),
+        defaultCompiler: compilerProps('defaultCompiler', ''),
+        compileOptions: compilerProps("options"),
         sources: sources
     };
     var text = "var OPTIONS = " + JSON.stringify(options) + ";";
@@ -193,8 +198,8 @@ function retryPromise(promiseFunc, name, maxFails, retryMs) {
 }
 
 function configuredCompilers() {
-    var exes = props.get("gcc-explorer", "compilers", "/usr/bin/g++").split(":");
-    var ndk = props.get('gcc-explorer', 'androidNdk');
+    var exes = compilerProps("compilers", "/usr/bin/g++").split(":");
+    var ndk = compilerProps('androidNdk');
     if (ndk) {
         var toolchains = fs.readdirSync(ndk + "/toolchains");
         toolchains.forEach(function (v, i, a) {
@@ -245,23 +250,23 @@ function configuredCompilers() {
                     });
                 },
                 host + ":" + port,
-                props.get('gcc-explorer', 'proxyRetries', 20),
-                props.get('gcc-explorer', 'proxyRetryMs', 500));
+                gccProps('proxyRetries', 20),
+                gccProps('proxyRetryMs', 500));
         }
         var base = "compiler." + name;
-        var exe = props.get("gcc-explorer", base + ".exe", "");
+        var exe = compilerProps(base + ".exe", "");
         if (!exe) {
             return Promise.resolve({id: name, exe: name, name: name});
         }
         return Promise.resolve({
             id: name,
             exe: exe,
-            name: props.get("gcc-explorer", base + ".name", name),
-            alias: props.get("gcc-explorer", base + ".alias"),
-            versionFlag: props.get("gcc-explorer", base + ".versionFlag"),
-            is6g: !!props.get("gcc-explorer", base + ".is6g", false),
-            intelAsm: props.get("gcc-explorer", base + ".intelAsm", ""),
-            supportsBinary: !!props.get("gcc-explorer", base + ".supportsBinary", false)
+            name: compilerProps(base + ".name", name),
+            alias: compilerProps(base + ".alias"),
+            versionFlag: compilerProps(base + ".versionFlag"),
+            is6g: !!compilerProps(base + ".is6g", false),
+            intelAsm: compilerProps(base + ".intelAsm", ""),
+            supportsBinary: !!compilerProps(base + ".supportsBinary", false)
         });
     }));
 }

etc/config/c++.defaults.properties

@@ -0,0 +1,9 @@
+# Default settings for C++
+compilers=/usr/bin/g++-4.4:/usr/bin/g++-4.5:/usr/bin/g++-4.6:/usr/bin/g++-4.7:/usr/bin/clang++:/usr/bin/g++
+compileFilename=example.cpp
+postProcess=c++filt
+#androidNdk=/opt/google/android-ndk-r9c
+options=-O2
+binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy)$
+stubRe=\bmain\b
+stubText=int main(int,char**){/*stub provided by Compiler Explorer*/}

etc/config/c++.lud-ldnmg01.properties

@@ -0,0 +1,18 @@
+# Default settings for GCC Explorer.
+defaultCompiler=g46
+compilers=g44:g45:g46:clang35:g51
+compiler.g44.exe=/usr/bin/g++-4.4
+compiler.g44.name=g++ 4.4
+compiler.g44.alias=/usr/bin/g++-4.4
+compiler.g45.exe=/usr/bin/g++-4.5
+compiler.g45.name=g++ 4.5
+compiler.g45.alias=/usr/bin/g++-4.5
+compiler.g46.exe=/usr/bin/g++-4.6
+compiler.g46.name=g++ 4.6
+compiler.g46.alias=/usr/bin/g++-4.6
+compiler.clang35.exe=/usr/bin/clang++-3.5
+compiler.clang35.name=Clang 3.5
+compiler.clang35.intelAsm=-mllvm --x86-asm-syntax=intel
+compiler.g51.exe=/home/mgodbolt/.fighome/runtime/gcc/5.1.0-1/bin/g++
+compiler.g51.name=g++ 5.1
+#compiler-wrapper=./c-preload/compiler-wrapper

etc/config/d.defaults.properties

@@ -0,0 +1,3 @@
+compilers=/usr/bin/gdc:/usr/bin/gdc-4.4:/usr/bin/gdc-4.6
+compileFilename=example.d
+postProcess=d/demangle

etc/config/gcc-explorer.defaults.properties

@@ -1,18 +1,12 @@
 # Default settings for GCC Explorer.
 port=10240
 compileTimeoutMs=1000
-compilers=/usr/bin/g++-4.4:/usr/bin/g++-4.5:/usr/bin/g++-4.6:/usr/bin/g++-4.7:/usr/bin/clang++:/usr/bin/g++
 defaultSource=builtin
-compileFilename=example.cpp
-postProcess=c++filt
 cacheMb=50
-#androidNdk=/opt/google/android-ndk-r9c
 language=C++
-options=-O2
 maxConcurrentCompiles=2
 staticMaxAgeMs=1000
 gcIntervalSecs=300
 gcHeapDiffEverySecs=3600
-binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy)$
-stubRe=\bmain\b
-stubText=int main(int,char**){/*stub provided by Compiler Explorer*/}
+optionsWhitelistRe=.*
+optionsBlacklistRe=^((-wrapper|-fplugin?|(@.*)|-I|-i)(=.*)?|--)$

etc/config/gcc-explorer.lud-ldnmg01.properties

@@ -1,39 +1 @@
-# Default settings for GCC Explorer.
-#port=10240
-compileTimeoutMs=5000
-defaultCompiler=g46
-#compilers=g44:g45:g46:clang35:g51
-compilers=go:6g
-language=go
-compileFilename=file.go
-compiler.go.exe=/usr/bin/gccgo
-compiler.go.name=GCC go
-compiler.6g.exe=/home/mgodbolt/build/go/pkg/tool/linux_amd64/6g
-compiler.6g.name=6g go
-compiler.6g.versionFlag=-V
-compiler.6g.is6g=true
-compiler.g44.exe=/usr/bin/g++-4.4
-compiler.g44.name=g++ 4.4
-compiler.g44.alias=/usr/bin/g++-4.4
-compiler.g45.exe=/usr/bin/g++-4.5
-compiler.g45.name=g++ 4.5
-compiler.g45.alias=/usr/bin/g++-4.5
-compiler.g46.exe=/usr/bin/g++-4.6
-compiler.g46.name=g++ 4.6
-compiler.g46.alias=/usr/bin/g++-4.6
-compiler.clang35.exe=/usr/bin/clang++-3.5
-compiler.clang35.name=Clang 3.5
-#compiler.clang35.intelAsm=-masm=intel
-compiler.clang35.intelAsm=-mllvm --x86-asm-syntax=intel
-compiler.g51.exe=/home/mgodbolt/.fighome/runtime/gcc/5.1.0-1/bin/g++
-compiler.g51.name=g++ 5.1
-#compilers=/usr/bin/g++-4.4:/usr/bin/g++-4.5:/usr/bin/g++-4.6:/usr/bin/g++-4.7:/usr/bin/clang++:/home/mgodbolt/apps/intel-icc-oss/bin/icc:/site/apps/gcc-4.7.2-drw.patched.6/bin/g++:/site/apps/gcc-4.8.0-drw.1/bin/g++
-#compiler-wrapper=./c-preload/compiler-wrapper
-#compilers=/usr/bin/gdc:/usr/bin/gdc-4.4:/usr/bin/gdc-4.6
-#compileFilename=example.d
-#postProcess=d/demangle
-#TODO seriously consider moving all the lang-specific options into <lang>.properties
-# for go:
-binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy|.*@plt)$
-stubRe=\bfunc\s+main\b
-stubText=func main() {/*stub provided by Compiler Explorer*/}
+language=C++

etc/config/gcc-explorer.rust.properties

@@ -1,10 +0,0 @@
-port=10242
-compileTimeoutMs=1500
-compilers=/usr/local/bin/rustc
-compileFilename=example.rs
-compileToAsm=--emit asm --crate-type staticlib
-#compiler-wrapper=./c-preload/compiler-wrapper
-#max-asm-size=262144
-#cacheMb=100
-language=Rust
-options=-O --crate-type staticlib

etc/config/gcc-explorer.test1.properties

@@ -1,4 +0,0 @@
-compilers=localhost@20480:g44
-compiler.g44.exe=/usr/bin/g++-4.4
-compiler.g44.name=g++ 4.4
-compiler.g44.alias=/usr/bin/g++-4.4

etc/config/gcc-explorer.test2.properties

@@ -1,9 +0,0 @@
-port=20480
-compilers=g45:g46:clang34
-compiler.g45.name=g++ 4.5
-compiler.g45.alias=/usr/bin/g++-4.5
-compiler.g46.exe=/usr/bin/g++-4.6
-compiler.g46.name=g++ 4.6
-compiler.g46.alias=/usr/bin/g++-4.6
-compiler.clang34.exe=/usr/bin/clang++-3.4
-compiler.clang34.name=Clang 4.6

etc/config/gcc-options.defaults.properties

@@ -1,2 +0,0 @@
-whitelistRe=.*
-blacklistRe=^((-wrapper|-fplugin?|(@.*)|-I|-i)(=.*)?|--)$

etc/config/go.defaults.properties

@@ -0,0 +1,4 @@
+compileFilename=file.go
+binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy|.*@plt)$
+stubRe=\bfunc\s+main\b
+stubText=func main() {/*stub provided by Compiler Explorer*/}

etc/config/go.lud-ldnmg01.properties

@@ -0,0 +1,7 @@
+compilers=go:6g
+compiler.go.exe=/usr/bin/gccgo
+compiler.go.name=GCC go
+compiler.6g.exe=/home/mgodbolt/build/go/pkg/tool/linux_amd64/6g
+compiler.6g.name=6g go
+compiler.6g.versionFlag=-V
+compiler.6g.is6g=true

etc/config/rust.defaults.properties

@@ -0,0 +1,9 @@
+compilers=/usr/local/bin/rustc
+options=-O --crate-type staticlib
+# TODO; remove need to have some kind of postprocess on everything
+postProcess=c++filt
+compileFilename=example.rs
+compileToAsm=--emit asm --crate-type staticlib
+stubRe=\bmain\b
+stubText=pub fn main() {/*stub provided by Compiler Explorer*/}
+binaryHideFuncRe=^(_.*|(de)?register_tm_clones|frame_dummy|.*@plt.*)$

lib/asm.js

@@ -23,8 +23,6 @@
 // POSSIBILITY OF SUCH DAMAGE.
 
 (function () {
-    var props = require('./properties.js');
-
     function processAsm(asm, filters) {
         if (filters.binary) return processBinaryAsm(asm, filters);
 
@@ -107,10 +105,12 @@
     }
 
     var binaryHideFuncRe = null;
-    function initialise() {
-        var pattern = props.get("gcc-explorer", 'binaryHideFuncRe');
+    var maxAsmLines = 500;
+    function initialise(compilerProps) {
+        var pattern = compilerProps('binaryHideFuncRe');
         console.log("asm: binary re = " + pattern);
         binaryHideFuncRe = new RegExp(pattern);
+        maxAsmLines = compilerProps('maxLinesOfAsm', maxAsmLines);
     }
     function isUserFunction(func) {
         return !func.match(binaryHideFuncRe);
@@ -126,6 +126,12 @@
         var source = null;
         var func = null;
         asmLines.forEach(function (line) {
+            if (result.length >= maxAsmLines) {
+                if (result.length == maxAsmLines) {
+                    result.push({text: "[truncated; too many lines]", source: null});
+                }
+                return;
+            }
             var match = line.match(lineRe);
             if (match) {
                 source = parseInt(match[2]);

lib/compile.js

@@ -22,8 +22,7 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 // POSSIBILITY OF SUCH DAMAGE.
 
-var props = require('./properties'),
-    child_process = require('child_process'),
+var child_process = require('child_process'),
     temp = require('temp'),
     path = require('path'),
     httpProxy = require('http-proxy'),
@@ -42,17 +41,21 @@ function periodicCleanup() {
         if (stats) console.log("Directory cleanup stats:", stats);
     });
 }
+var gccProps = null;
+var compilerProps = null;
 var stubRe = null;
 var stubText = null;
-setTimeout(function () {
-    // property library isn't initialized until after the first tick
-    var tempDirCleanupSecs = props.get("gcc-explorer", "tempDirCleanupSecs", 600);
+
+function initialise(gccProps_, compilerProps_) {
+    gccProps = gccProps_;
+    compilerProps = compilerProps_;
+    var tempDirCleanupSecs = gccProps("tempDirCleanupSecs", 600);
     console.log("Cleaning temp dirs every " + tempDirCleanupSecs + " secs");
     setInterval(periodicCleanup, tempDirCleanupSecs * 1000);
-    asm.initialise();
-    stubRe = props.get("gcc-explorer", "stubRe");
-    stubText = props.get("gcc-explorer", "stubText");
-}, 0);
+    asm.initialise(compilerProps);
+    stubRe = compilerProps("stubRe");
+    stubText = compilerProps("stubText");
+}
 
 function Compile(compilers) {
     this.compilersById = {};
@@ -60,17 +63,17 @@ function Compile(compilers) {
     compilers.forEach(function (compiler) {
         self.compilersById[compiler.id] = compiler;
     });
-    this.okOptions = new RegExp(props.get('gcc-options', 'whitelistRe', '.*'));
-    this.badOptions = new RegExp(props.get('gcc-options', 'blacklistRe'));
+    this.okOptions = new RegExp(gccProps('optionsWhitelistRe', '.*'));
+    this.badOptions = new RegExp(gccProps('optionsBlacklistRe'));
     this.cache = LRU({
-        max: props.get('gcc-explorer', 'cacheMb') * 1024 * 1024,
+        max: gccProps('cacheMb') * 1024 * 1024,
         length: function (n) {
             return n.asm.length;
         }
     });
     this.cacheHits = 0;
     this.cacheMisses = 0;
-    this.compileQueue = new Queue(props.get("gcc-explorer", "maxConcurrentCompiles", 1), Infinity);
+    this.compileQueue = new Queue(gccProps("maxConcurrentCompiles", 1), Infinity);
 }
 
 Compile.prototype.newTempDir = function () {
@@ -132,7 +135,7 @@ Compile.prototype.runCompiler = function (compiler, options) {
         okToCache = false;
         child.kill();
         stderr += "\nKilled - processing time exceeded";
-    }, props.get("gcc-explorer", "compileTimeoutMs", 100));
+    }, gccProps("compileTimeoutMs", 100));
     child.stdout.on('data', function (data) {
         stdout += data;
     });
@@ -215,7 +218,7 @@ Compile.prototype.compile = function (source, compiler, options, filters) {
 
     var tempFileAndDirPromise = Promise.resolve().then(function () {
         return self.newTempDir().then(function (dirPath) {
-            var inputFilename = path.join(dirPath, props.get("gcc-explorer", "compileFilename"));
+            var inputFilename = path.join(dirPath, compilerProps("compileFilename"));
             return self.writeFile(inputFilename, source).then(function () {
                 return {inputFilename: inputFilename, dirPath: dirPath};
             });
@@ -224,26 +227,26 @@ Compile.prototype.compile = function (source, compiler, options, filters) {
     var compileToAsmPromise = tempFileAndDirPromise.then(function (info) {
         var inputFilename = info.inputFilename;
         var dirPath = info.dirPath;
-        var postProcess = props.get("gcc-explorer", "postProcess");
+        var postProcess = compilerProps("postProcess");
         var outputFilename = path.join(dirPath, 'output.S');
         if (compilerInfo.intelAsm && filters.intel && !filters.binary) {
             options = options.concat(compilerInfo.intelAsm.split(" "));
         }
         var compileToAsm;
         if (!filters.binary) {
-            compileToAsm = props.get("gcc-explorer", "compileToAsm", "-S").split(" ");
+            compileToAsm = compilerProps("compileToAsm", "-S").split(" ");
         } else {
             compileToAsm = [];
         }
         options = options.concat(['-g', '-o', outputFilename]).concat(compileToAsm).concat([inputFilename]);
 
         var compilerExe = compilerInfo.exe;
-        var compilerWrapper = props.get("gcc-explorer", "compiler-wrapper");
+        var compilerWrapper = compilerProps("compiler-wrapper");
         if (compilerWrapper) {
             options = [compilerExe].concat(options);
             compilerExe = compilerWrapper;
         }
-        var maxSize = props.get("gcc-explorer", "max-asm-size", 8 * 1024 * 1024);
+        var maxSize = gccProps("max-asm-size", 8 * 1024 * 1024);
         return self.runCompiler(compilerExe, options).then(function (result) {
             result.dirPath = dirPath;
             if (result.code !== 0) {
@@ -364,5 +367,6 @@ function compileHandler(compilers) {
 }
 
 module.exports = {
-    compileHandler: compileHandler
+    compileHandler: compileHandler,
+    initialise: initialise
 };

lib/properties.js

@@ -28,13 +28,15 @@ var properties = {};
 
 var hierarchy = [];
 
+var propDebug = false;
+
 function findProps(base, elem) {
     var name = base + '.' + elem;
     return properties[name];
 }
 
 function debug(string) {
-    //console.log("prop: " + string);
+    if (propDebug) console.log("prop: " + string);
 }
 
 function get(base, property, defaultValue) {
@@ -93,7 +95,15 @@ function initialize(directory, hier) {
     });
 }
 
+function propsFor(base) {
+    return function(property, defaultValue) {
+        return get(base, property, defaultValue);
+    }
+}
+
 module.exports = {
     get: get,
-    initialize: initialize
+    propsFor: propsFor,
+    initialize: initialize,
+    setDebug: function(debug) { propDebug = debug; }
 };

lib/sources/builtin.js

@@ -27,7 +27,7 @@
     var props = require('../properties.js'),
         path = require('path'),
         fs = require('fs');
-
+//TODO update this to use compiler options?
     var sourcePath = props.get('builtin', 'sourcepath', './examples/c++');
     var sourceMatch = new RegExp(props.get('builtin', 'extensionRe', '.*\\.cpp$'));
     var examples = fs.readdirSync(sourcePath)

static/compiler.js

@@ -343,7 +343,8 @@ function Compiler(domRoot, origFilters, windowLocalPrefix, onChangeCallback, lan
         var compiler = compilersById[$('.compiler').val()];
         if (compiler === undefined)
             return;
-        domRoot.find('.filter button.btn[value="intel"]').toggleClass("disabled", !compiler.intelAsm);
+        var supportsIntel = compiler.asm || filters.binary;  // TODO: separate binary so it has its own buttons and then this problem goes away
+        domRoot.find('.filter button.btn[value="intel"]').toggleClass("disabled", !supportsIntel);
         $(".compilerVersion").text(compiler.name + " (" + compiler.version + ")");
     }