okToCache fix for nsjail errors and extra checks (#8148)

etc/config/c++.amazon.properties

@@ -1,4 +1,4 @@
-compilers=&gcc86:&icc:&icx:&clang:&clangx86trunk:&clang-rocm:&mosclang-trunk:&rvclang:&wasmclang:&loongarch-clang:&cross:&ellcc:&zapcc:&djggp:&armclang32:&armclang64:&zigcxx:&cxx6502:&nvcxx_arm_cxx:godbolt.org@443/gpu:godbolt.org@443/winprod:&hexagon-clang:&edg:&vast:&qnx:&z80-clang:&clad-clang
+compilers=&gcc86:&icc:&icx:&clang:&clangx86trunk:&clang-rocm:&mosclang-trunk:&rvclang:&wasmclang:&loongarch-clang:&cross:&ellcc:&zapcc:&djggp:&armclang32:&armclang64:&zigcxx:&cxx6502:godbolt.org@443/gpu:godbolt.org@443/winprod:&hexagon-clang:&edg:&vast:&qnx:&z80-clang:&clad-clang
 # Disabled: nvcxx_x86_cxx
 # The disabled groups are actually used in the c++.gpu.properties. One day these might exist on both servers, so I want
 # to keep them in the same place.
@@ -4538,93 +4538,6 @@ compiler.nvcxx_x86_cxx25_9.nvdisasm=/opt/compiler-explorer/hpc_sdk/Linux_x86_64/
 compiler.nvcxx_x86_cxx25_9.exe=/opt/compiler-explorer/hpc_sdk/Linux_x86_64/25.9/compilers/bin/nvc++
 compiler.nvcxx_x86_cxx25_9.semver=25.9
 
-group.nvcxx_arm_cxx.compilers=nvcxx_arm_cxx22_11:nvcxx_arm_cxx23_1:nvcxx_arm_cxx23_3:nvcxx_arm_cxx23_5:nvcxx_arm_cxx23_7:nvcxx_arm_cxx23_9:nvcxx_arm_cxx23_11:nvcxx_arm_cxx24_1:nvcxx_arm_cxx24_3:nvcxx_arm_cxx24_5:nvcxx_arm_cxx24_7:nvcxx_arm_cxx24_9:nvcxx_arm_cxx24_11:nvcxx_arm_cxx25_1:nvcxx_arm_cxx25_3:nvcxx_arm_cxx25_5:nvcxx_arm_cxx25_7:nvcxx_arm_cxx25_9
-group.nvcxx_arm_cxx.options=
-group.nvcxx_arm_cxx.supportsBinary=true
-group.nvcxx_arm_cxx.binaryHideFuncRe=^(__.*|_(init|start|fini)|(de)?register_tm_clones|call_gmon_start|frame_dummy|\.plt.*|_dl_relocate_static_pie)$
-group.nvcxx_arm_cxx.needsMulti=false
-group.nvcxx_arm_cxx.stubRe=\bmain\b
-group.nvcxx_arm_cxx.stubText=int main(void){return 0;/*stub provided by Compiler Explorer*/}
-group.nvcxx_arm_cxx.supportsExecute=true
-group.nvcxx_arm_cxx.supportsLibraryCodeFilter=true
-group.nvcxx_arm_cxx.demanglerType=nvhpc
-group.nvcxx_arm_cxx.groupName=nvc++ arm
-group.nvcxx_arm_cxx.baseName=ARM64 nvc++
-group.nvcxx_arm_cxx.isSemVer=true
-group.nvcxx_arm_cxx.instructionSet=aarch64
-group.nvcxx_arm_cxx.compilerCategories=nvc++
-
-compiler.nvcxx_arm_cxx22_11.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/22.11/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx22_11.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/22.11/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx22_11.semver=22.11
-
-compiler.nvcxx_arm_cxx23_1.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.1/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx23_1.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.1/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx23_1.semver=23.1
-
-compiler.nvcxx_arm_cxx23_3.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.3/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx23_3.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.3/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx23_3.semver=23.3
-
-compiler.nvcxx_arm_cxx23_5.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.5/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx23_5.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.5/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx23_5.semver=23.5
-
-compiler.nvcxx_arm_cxx23_7.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.7/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx23_7.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.7/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx23_7.semver=23.7
-
-compiler.nvcxx_arm_cxx23_9.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.9/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx23_9.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.9/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx23_9.semver=23.9
-
-compiler.nvcxx_arm_cxx23_11.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.11/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx23_11.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/23.11/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx23_11.semver=23.11
-
-compiler.nvcxx_arm_cxx24_1.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.1/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx24_1.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.1/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx24_1.semver=24.1
-
-compiler.nvcxx_arm_cxx24_3.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx24_3.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx24_3.semver=24.3
-
-compiler.nvcxx_arm_cxx24_5.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.5/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx24_5.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.5/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx24_5.semver=24.5
-
-compiler.nvcxx_arm_cxx24_7.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.7/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx24_7.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.7/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx24_7.semver=24.7
-
-compiler.nvcxx_arm_cxx24_9.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.9/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx24_9.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.9/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx24_9.semver=24.9
-
-compiler.nvcxx_arm_cxx24_11.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.11/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx24_11.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/24.11/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx24_11.semver=24.11
-
-compiler.nvcxx_arm_cxx25_1.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.1/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx25_1.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.1/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx25_1.semver=25.1
-
-compiler.nvcxx_arm_cxx25_3.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.3/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx25_3.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.3/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx25_3.semver=25.3
-
-compiler.nvcxx_arm_cxx25_5.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.5/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx25_5.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.5/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx25_5.semver=25.5
-
-compiler.nvcxx_arm_cxx25_7.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.7/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx25_7.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.7/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx25_7.semver=25.7
-
-compiler.nvcxx_arm_cxx25_9.demangler=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.9/compilers/bin/nvdecode
-compiler.nvcxx_arm_cxx25_9.exe=/opt/compiler-explorer/hpc_sdk/Linux_aarch64/25.9/compilers/bin/nvc++
-compiler.nvcxx_arm_cxx25_9.semver=25.9
 
 #################################
 # EDG compiler

etc/config/c.amazon.properties

@@ -774,7 +774,7 @@ compiler.armv8-full-cclang-trunk.alias=armv8.5-cclang-trunk
 
 ################################
 # LLVM-MOS (6502 Clang)
-group.cmosclang-trunk.compilers=cmos-nes-cnrom-trunk:cmos-nes-mmc1-trunk:cmos-nes-mmc3-trunk:cmos-nes-nrom-trunk:cmos-atari8-trunk:cmos-cx16-trunk:cmos-c64-trunk:cmos-mega65-trunk:cmos-osi-c1p-trunk
+group.cmosclang-trunk.compilers=cmos-nes-cnrom-trunk:cmos-nes-mmc1-trunk:cmos-nes-mmc3-trunk:cmos-nes-nrom-trunk:cmos-cx16-trunk:cmos-c64-trunk:cmos-mega65-trunk:cmos-osi-c1p-trunk
 group.cmosclang-trunk.baseName=llvm-mos
 group.cmosclang-trunk.groupName=llvm-mos (6502) clang
 group.cmosclang-trunk.options=-fno-lto -mllvm -zp-avail=224
@@ -802,9 +802,6 @@ compiler.cmos-nes-mmc3-trunk.semver=nes-mmc3
 compiler.cmos-nes-nrom-trunk.exe=/opt/compiler-explorer/llvm-mos-trunk/bin/mos-nes-nrom-clang
 compiler.cmos-nes-nrom-trunk.semver=nes-nrom
 
-compiler.cmos-atari8-trunk.exe=/opt/compiler-explorer/llvm-mos-trunk/bin/mos-atari8-clang
-compiler.cmos-atari8-trunk.semver=atari 8-bit
-
 compiler.cmos-cx16-trunk.exe=/opt/compiler-explorer/llvm-mos-trunk/bin/mos-cx16-clang
 compiler.cmos-cx16-trunk.semver=commander X16
 

etc/config/elixir.amazon.properties

@@ -0,0 +1 @@
+compilers=

etc/config/go.amazon.properties

@@ -452,8 +452,8 @@ compiler.ppc64le_gl122.exe=/opt/compiler-explorer/golang-1.22.1/go/bin/go
 compiler.ppc64le_gl122.semver=1.22.12
 compiler.ppc64le_gl123.exe=/opt/compiler-explorer/golang-1.23.8/go/bin/go
 compiler.ppc64le_gl123.semver=1.23.8
-compiler.ppc64le_gl124.exe=/opt/compiler-explorer/golang-1.2+.2/go/bin/go
+compiler.ppc64le_gl124.exe=/opt/compiler-explorer/golang-1.24.2/go/bin/go
-compiler.ppc64le_gl124.semver=1.2+.2
+compiler.ppc64le_gl124.semver=1.24.2
 compiler.ppc64le_gltip.exe=/opt/compiler-explorer/go-tip/bin/go
 compiler.ppc64le_gltip.semver=(tip)
 compiler.ppc64le_gltip.isNightly=true

etc/config/jakt.amazon.properties

@@ -1,5 +1,5 @@
-compilers=selfhosted
+compilers=
-defaultCompiler=selfhosted
+defaultCompiler=
 
 externalparser=CEAsmParser
 externalparser.exe=/usr/local/bin/asm-parser
@@ -13,7 +13,3 @@ compilerType=jakt
 licenseName=BSD 2-Clause "Simplified" License
 licenseLink=https://github.com/SerenityOS/jakt/blob/main/LICENSE
 licensePreamble=Copyright (c) 2022, JT, Andreas Kling. All rights reserved.
-
-compiler.selfhosted.exe=/opt/compiler-explorer/jakt-trunk/bin/jakt
-compiler.selfhosted.name=jakt (trunk)
-compiler.selfhosted.options=--runtime-path /opt/compiler-explorer/jakt-trunk/runtime --prettify-cpp-source --clang-format-path /opt/compiler-explorer/clang-trunk/bin/clang-format --dot-clang-format-path /opt/compiler-explorer/jakt-trunk/.clang-format --cxx-compiler-path /opt/compiler-explorer/gcc-12.1.0/bin/g++

etc/config/toit.amazon.properties

@@ -0,0 +1 @@
+compilers=

lib/app.interfaces.ts

@@ -35,6 +35,7 @@ export type AppArguments = {
     doCache: boolean;
     fetchCompilersFromRemote: boolean;
     ensureNoCompilerClash: boolean | undefined;
+    exitOnCompilerFailure?: boolean;
     prediscovered?: string;
     discoveryOnly?: string;
     staticPath?: string;

lib/app/cli.ts

@@ -64,6 +64,7 @@ export interface CompilerExplorerOptions {
     language?: string[];
     cache: boolean;
     ensureNoIdClash?: boolean;
+    exitOnCompilerFailure?: boolean;
     logHost?: string;
     logPort?: number;
     hostnameForLogging?: string;
@@ -101,6 +102,7 @@ export function parseCommandLine(argv: string[]): CompilerExplorerOptions {
         .option('--language <languages...>', 'Only load specified languages for faster startup')
         .option('--no-cache', 'Do not use caching for compilation results')
         .option('--ensure-no-id-clash', "Don't run if compilers have clashing ids")
+        .option('--exit-on-compiler-failure', 'Exit with error code if any compilers fail to initialize')
         .option('--logHost, --log-host <hostname>', 'Hostname for remote logging')
         .option('--logPort, --log-port <port>', 'Port for remote logging', parsePortNumberForOptions)
         .option('--hostnameForLogging, --hostname-for-logging <hostname>', 'Hostname to use in logs')
@@ -188,6 +190,7 @@ export function convertOptionsToAppArguments(
         doCache: options.cache,
         fetchCompilersFromRemote: options.remoteFetch,
         ensureNoCompilerClash: options.ensureNoIdClash,
+        exitOnCompilerFailure: options.exitOnCompilerFailure,
         prediscovered: options.prediscovered,
         discoveryOnly: options.discoveryOnly,
         staticPath: options.static,

lib/app/compilation-env.ts

@@ -56,7 +56,7 @@ export async function initializeCompilationEnvironment(
         appArgs.doCache,
     );
 
-    const compileHandler = new CompileHandler(compilationEnvironment, awsProps);
+    const compileHandler = new CompileHandler(compilationEnvironment, awsProps, appArgs);
     compilationEnvironment.setCompilerFinder(compileHandler.findCompiler.bind(compileHandler));
 
     return {

lib/exec.ts

@@ -47,6 +47,7 @@ type NsJailOptions = {
 };
 
 const execProps = propsFor('execution');
+const c_nsjail_permissions_error = 'runChild():486 Launching child process failed';
 
 let stdbufPath: null | string = null;
 
@@ -352,15 +353,41 @@ export function getExecuteCEWrapperOptions(command: string, args: string[], opti
     return getCeWrapperOptions('execute', command, args, options);
 }
 
-function sandboxNsjail(command: string, args: string[], options: ExecutionOptions) {
+function hasNsjailPermissionsIssue(stderr: string): boolean {
-    logger.info('Sandbox execution via nsjail', {command, args});
+    return stderr.includes(c_nsjail_permissions_error);
-    const nsOpts = getSandboxNsjailOptions(command, args, options);
-    return executeDirect(execProps<string>('nsjail'), nsOpts.args, nsOpts.options, nsOpts.filenameTransform);
 }
 
-function executeNsjail(command: string, args: string[], options: ExecutionOptions) {
+async function sandboxNsjail(
+    command: string,
+    args: string[],
+    options: ExecutionOptions,
+): Promise<UnprocessedExecResult> {
+    logger.info('Sandbox execution via nsjail', {command, args});
+    const nsOpts = getSandboxNsjailOptions(command, args, options);
+    const result = await executeDirect(
+        execProps<string>('nsjail'),
+        nsOpts.args,
+        nsOpts.options,
+        nsOpts.filenameTransform,
+    );
+    if (hasNsjailPermissionsIssue(result.stderr)) result.okToCache = false;
+    return result;
+}
+
+async function executeNsjail(
+    command: string,
+    args: string[],
+    options: ExecutionOptions,
+): Promise<UnprocessedExecResult> {
     const nsOpts = getNsJailOptions('execute', command, args, options);
-    return executeDirect(execProps<string>('nsjail'), nsOpts.args, nsOpts.options, nsOpts.filenameTransform);
+    const result = await executeDirect(
+        execProps<string>('nsjail'),
+        nsOpts.args,
+        nsOpts.options,
+        nsOpts.filenameTransform,
+    );
+    if (hasNsjailPermissionsIssue(result.stderr)) result.okToCache = false;
+    return result;
 }
 
 function sandboxCEWrapper(command: string, args: string[], options: ExecutionOptions) {

lib/handlers/compile.ts

@@ -24,6 +24,7 @@
 
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import process from 'node:process';
 import * as Sentry from '@sentry/node';
 import express from 'express';
 import Server from 'http-proxy';
@@ -46,6 +47,7 @@ import {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfa
 import {LanguageKey} from '../../types/languages.interfaces.js';
 import {SelectedLibraryVersion} from '../../types/libraries/libraries.interfaces.js';
 import {ResultLine} from '../../types/resultline/resultline.interfaces.js';
+import {AppArguments} from '../app.interfaces.js';
 import {BaseCompiler} from '../base-compiler.js';
 import {parseExecutionParameters, parseTools, parseUserArguments} from '../compilation/compilation-request-parser.js';
 import {CompilationEnvironment} from '../compilation-env.js';
@@ -129,6 +131,7 @@ export class CompileHandler implements ICompileHandler {
     private readonly textBanner: string;
     private readonly proxy: Server;
     private readonly awsProps: PropertyGetter;
+    private readonly appArgs: AppArguments | undefined;
     private clientOptions: ClientOptionsType | null = null;
     private readonly compileCounter = new Counter({
         name: 'ce_compilations_total',
@@ -151,11 +154,12 @@ export class CompileHandler implements ICompileHandler {
         labelNames: ['language'],
     });
 
-    constructor(compilationEnvironment: CompilationEnvironment, awsProps: PropertyGetter) {
+    constructor(compilationEnvironment: CompilationEnvironment, awsProps: PropertyGetter, appArgs?: AppArguments) {
         this.compilerEnv = compilationEnvironment;
         this.textBanner = this.compilerEnv.ceProps<string>('textBanner');
         this.proxy = Server.createProxyServer({});
         this.awsProps = awsProps;
+        this.appArgs = appArgs;
         initialise(this.compilerEnv);
 
         // Mostly cribbed from
@@ -295,7 +299,8 @@ export class CompileHandler implements ICompileHandler {
         const compilersById: Partial<Record<LanguageKey, Record<string, BaseCompiler>>> = {};
         try {
             this.clientOptions = clientOptions;
-            logger.info('Creating compilers: ' + compilers.length);
+            const totalCompilers = compilers.length;
+            logger.info('Creating compilers: ' + totalCompilers);
             let compilersCreated = 0;
             const createdCompilers = remove(await Promise.all(compilers.map(c => this.create(c))), null);
             for (const compiler of createdCompilers) {
@@ -304,6 +309,17 @@ export class CompileHandler implements ICompileHandler {
                 compilersById[langId][compiler.getInfo().id] = compiler;
                 compilersCreated++;
             }
+
+            const failedCount = totalCompilers - compilersCreated;
+            if (failedCount > 0) {
+                logger.error(`Failed to create ${failedCount} out of ${totalCompilers} compilers`);
+
+                if (this.appArgs?.exitOnCompilerFailure) {
+                    logger.error('Exiting due to compiler creation failures (exitOnCompilerFailure=true)');
+                    process.exit(1);
+                }
+            }
+
             logger.info('Compilers created: ' + compilersCreated);
             if (this.awsProps) {
                 logger.info('Fetching possible arguments from storage');