From 4fe8397fac974294f2bb6a17476201b9e6a5a84d Mon Sep 17 00:00:00 2001 From: Patrick Quist Date: Sat, 26 Oct 2024 17:42:22 +0200 Subject: [PATCH] Remote execution (#6700) --- app.ts | 26 +- .../compiler-explorer.aarch64prod.properties | 9 + ...ompiler-explorer.aarch64staging.properties | 9 + .../compiler-explorer.amazon.properties | 5 + .../compiler-explorer.amazonwin.properties | 4 + .../compiler-explorer.staging.properties | 3 + etc/config/execution.aarch64.properties | 5 + etc/config/rust.amazon.properties | 1 + lib/base-compiler.ts | 246 ++++-- lib/binaries/binary-utils.ts | 128 ++++ lib/{ => binaries}/win-utils.ts | 9 +- lib/compilation-env.ts | 24 +- lib/compilers/cerberus.ts | 5 +- lib/compilers/java.ts | 6 +- lib/compilers/kotlin.ts | 5 +- lib/compilers/rust.ts | 30 +- lib/compilers/win32-mingw-clang.ts | 2 +- lib/compilers/win32-mingw-gcc.ts | 2 +- lib/execution/base-execution-env.ts | 25 +- lib/execution/base-execution-triple.ts | 80 ++ lib/execution/events-websocket.ts | 134 ++++ lib/execution/execution-query.ts | 95 +++ lib/execution/execution-triple.ts | 131 ++++ lib/execution/remote-execution-env.ts | 110 +++ lib/execution/sqs-execution-queue.ts | 185 +++++ lib/handlers/compile.interfaces.ts | 4 +- lib/handlers/compile.ts | 3 +- lib/handlers/health-check.ts | 3 +- lib/instructionsets.ts | 4 + lib/properties.ts | 6 +- lib/toolchain-utils.ts | 2 +- package-lock.json | 718 ++++++++++++++++++ package.json | 2 + static/panes/compiler.ts | 6 +- static/panes/executor.ts | 15 +- test/base-compiler-tests.ts | 68 +- test/compilation-env.ts | 8 +- test/execution-triple-tests.ts | 78 ++ test/handlers/health-check-tests.ts | 26 +- test/library-tests.ts | 14 +- .../rust/rust.local.properties | 2 + test/tool-tests.ts | 8 +- test/utils.ts | 2 +- types/compilation/compilation.interfaces.ts | 8 +- types/instructionsets.ts | 1 + 45 files changed, 2121 insertions(+), 136 deletions(-) create mode 100644 etc/config/compiler-explorer.aarch64prod.properties create mode 100644 etc/config/compiler-explorer.aarch64staging.properties create mode 100644 etc/config/execution.aarch64.properties create mode 100644 lib/binaries/binary-utils.ts rename lib/{ => binaries}/win-utils.ts (95%) create mode 100644 lib/execution/base-execution-triple.ts create mode 100644 lib/execution/events-websocket.ts create mode 100644 lib/execution/execution-query.ts create mode 100644 lib/execution/execution-triple.ts create mode 100644 lib/execution/remote-execution-env.ts create mode 100644 lib/execution/sqs-execution-queue.ts create mode 100644 test/execution-triple-tests.ts create mode 100644 test/test-properties/rust/rust.local.properties diff --git a/app.ts b/app.ts index f0b90e074..304a4f2ae 100755 --- a/app.ts +++ b/app.ts @@ -53,6 +53,9 @@ import {CompilationEnvironment} from './lib/compilation-env.js'; import {CompilationQueue} from './lib/compilation-queue.js'; import {CompilerFinder} from './lib/compiler-finder.js'; import {startWineInit} from './lib/exec.js'; +import {RemoteExecutionQuery} from './lib/execution/execution-query.js'; +import {initHostSpecialties} from './lib/execution/execution-triple.js'; +import {startExecutionWorkerThread} from './lib/execution/sqs-execution-queue.js'; import {CompileHandler} from './lib/handlers/compile.js'; import * as healthCheck from './lib/handlers/health-check.js'; import {NoScriptHandler} from './lib/handlers/noscript.js'; @@ -522,9 +525,16 @@ async function main() { startWineInit(); + RemoteExecutionQuery.initRemoteExecutionArchs(ceProps, defArgs.env); + const clientOptionsHandler = new ClientOptionsHandler(sources, compilerProps, defArgs); const compilationQueue = CompilationQueue.fromProps(compilerProps.ceProps); - const compilationEnvironment = new CompilationEnvironment(compilerProps, compilationQueue, defArgs.doCache); + const compilationEnvironment = new CompilationEnvironment( + compilerProps, + awsProps, + compilationQueue, + defArgs.doCache, + ); const compileHandler = new CompileHandler(compilationEnvironment, awsProps); const storageType = getStorageTypeByKey(storageSolution); const storageHandler = new storageType(httpRoot, compilerProps, awsProps); @@ -538,6 +548,8 @@ async function main() { let initialCompilers: CompilerInfo[]; let prevCompilers; + const isExecutionWorker = ceProps('execqueue.is_worker', false); + if (opts.prediscovered) { const prediscoveredCompilersJson = await fs.readFile(opts.prediscovered, 'utf8'); initialCompilers = JSON.parse(prediscoveredCompilersJson); @@ -548,7 +560,7 @@ async function main() { } else { const initialFindResults = await compilerFinder.find(); initialCompilers = initialFindResults.compilers; - if (initialCompilers.length === 0) { + if (!isExecutionWorker && initialCompilers.length === 0) { throw new Error('Unexpected failure, no compilers found!'); } if (defArgs.ensureNoCompilerClash) { @@ -653,7 +665,8 @@ async function main() { // Handle healthchecks at the root, as they're not expected from the outside world .use( '/healthcheck', - new healthCheck.HealthCheckHandler(compilationQueue, healthCheckFilePath, compileHandler).handle, + new healthCheck.HealthCheckHandler(compilationQueue, healthCheckFilePath, compileHandler, isExecutionWorker) + .handle, ) .use(httpRoot, router) .use((req, res, next) => { @@ -847,6 +860,13 @@ async function main() { logger.info(' with disabled caching'); } setupEventLoopLagLogging(); + + if (isExecutionWorker) { + await initHostSpecialties(); + + startExecutionWorkerThread(ceProps, awsProps, compilationEnvironment); + } + startListening(webServer); } diff --git a/etc/config/compiler-explorer.aarch64prod.properties b/etc/config/compiler-explorer.aarch64prod.properties new file mode 100644 index 000000000..33f21fbfa --- /dev/null +++ b/etc/config/compiler-explorer.aarch64prod.properties @@ -0,0 +1,9 @@ +httpRoot=/aarch64prod +restrictToLanguages=none + +heaptrackPath=/opt/compiler-explorer/heaptrack-aarch64-v1.3.0 +libSegFaultPath=/opt/compiler-explorer/glibc-tools-arm64 + +execqueue.queue_url=https://sqs.us-east-1.amazonaws.com/052730242331/prod-execqueue +execqueue.events_url=wss://events.compiler-explorer.com/prod +execqueue.is_worker=true diff --git a/etc/config/compiler-explorer.aarch64staging.properties b/etc/config/compiler-explorer.aarch64staging.properties new file mode 100644 index 000000000..ee54b62fb --- /dev/null +++ b/etc/config/compiler-explorer.aarch64staging.properties @@ -0,0 +1,9 @@ +httpRoot=/aarch64staging +restrictToLanguages=none + +heaptrackPath=/opt/compiler-explorer/heaptrack-aarch64-v1.3.0 +libSegFaultPath=/opt/compiler-explorer/glibc-tools-arm64 + +execqueue.queue_url=https://sqs.us-east-1.amazonaws.com/052730242331/staging-execqueue +execqueue.events_url=wss://events.compiler-explorer.com/staging +execqueue.is_worker=true diff --git a/etc/config/compiler-explorer.amazon.properties b/etc/config/compiler-explorer.amazon.properties index f56a099c0..5a4d98906 100644 --- a/etc/config/compiler-explorer.amazon.properties +++ b/etc/config/compiler-explorer.amazon.properties @@ -76,3 +76,8 @@ compilationStatsNotifier=S3(compiler-explorer-logs,compile-stats,us-east-1,15m) compilationStaleAfterMs=60000 compilerVersionsUrl=https://api.compiler-explorer.com/get_deployed_exe_version + +execqueue.remote_archs_url=https://api.compiler-explorer.com/get_remote_execution_archs +execqueue.queue_url=https://sqs.us-east-1.amazonaws.com/052730242331/prod-execqueue +execqueue.events_url=wss://events.compiler-explorer.com/prod +execqueue.is_worker=false diff --git a/etc/config/compiler-explorer.amazonwin.properties b/etc/config/compiler-explorer.amazonwin.properties index 4a318aa1c..96b96b9f6 100644 --- a/etc/config/compiler-explorer.amazonwin.properties +++ b/etc/config/compiler-explorer.amazonwin.properties @@ -44,3 +44,7 @@ formatter.clangformat.styles=Google:LLVM:Mozilla:Chromium:WebKit:Microsoft:GNU formatter.clangformat.type=clangformat compilationStatsNotifier=S3(compiler-explorer-logs,compile-stats,us-east-1,15m) + +execqueue.queue_url=https://sqs.us-east-1.amazonaws.com/052730242331/prod-execqueue +execqueue.events_url=wss://events.compiler-explorer.com/prod +execqueue.is_worker=false diff --git a/etc/config/compiler-explorer.staging.properties b/etc/config/compiler-explorer.staging.properties index 07dde58ea..f8dbf932c 100644 --- a/etc/config/compiler-explorer.staging.properties +++ b/etc/config/compiler-explorer.staging.properties @@ -2,3 +2,6 @@ extraBodyClass=staging httpRoot=/staging motdUrl=/motd/motd-staging.json sentryEnvironment=staging + +execqueue.queue_url=https://sqs.us-east-1.amazonaws.com/052730242331/staging-execqueue +execqueue.events_url=wss://events.compiler-explorer.com/staging diff --git a/etc/config/execution.aarch64.properties b/etc/config/execution.aarch64.properties new file mode 100644 index 000000000..d0ce6bf04 --- /dev/null +++ b/etc/config/execution.aarch64.properties @@ -0,0 +1,5 @@ +sandboxType=nsjail +executionType=nsjail +wine= +wineServer= +firejail=/usr/local/firejail-0.9.70/bin/firejail diff --git a/etc/config/rust.amazon.properties b/etc/config/rust.amazon.properties index 0dacb9473..4650cf64d 100644 --- a/etc/config/rust.amazon.properties +++ b/etc/config/rust.amazon.properties @@ -1,6 +1,7 @@ compilers=&rust:&rustgcc:&mrustc:&rustccggcc objdumper=/opt/compiler-explorer/gcc-14.1.0/bin/objdump linker=/opt/compiler-explorer/gcc-14.1.0/bin/gcc +aarch64linker=/opt/compiler-explorer/arm64/gcc-14.1.0/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc defaultCompiler=r1820 demangler=/opt/compiler-explorer/gcc-14.1.0/bin/c++filt demanglerArgs=--format=rust diff --git a/lib/base-compiler.ts b/lib/base-compiler.ts index 48d757b46..be3e13ede 100644 --- a/lib/base-compiler.ts +++ b/lib/base-compiler.ts @@ -96,8 +96,12 @@ import { import {BaseDemangler, getDemanglerTypeByKey} from './demangler/index.js'; import {LLVMIRDemangler} from './demangler/llvm.js'; import * as exec from './exec.js'; +import {BaseExecutionTriple} from './execution/base-execution-triple.js'; import {IExecutionEnvironment} from './execution/execution-env.interfaces.js'; +import {RemoteExecutionQuery} from './execution/execution-query.js'; +import {matchesCurrentHost} from './execution/execution-triple.js'; import {getExecutionEnvironmentByKey} from './execution/index.js'; +import {RemoteExecutionEnvironment} from './execution/remote-execution-env.js'; import {ExternalParserBase} from './external-parsers/base.js'; import {getExternalParserByKey} from './external-parsers/index.js'; import {ParsedRequest} from './handlers/compile.js'; @@ -124,6 +128,7 @@ import { getSysrootByToolchainPath, getToolchainFlagFromOptions, getToolchainPath, + getToolchainPathWithOptionsArr, hasSysrootArg, hasToolchainArg, removeToolchainArg, @@ -988,7 +993,7 @@ export class BaseCompiler implements ICompiler { ); } - getSharedLibraryPathsAsLdLibraryPathsForExecution(libraries: SelectedLibraryVersion[], dirPath: string): string[] { + getSharedLibraryPathsAsLdLibraryPathsForExecution(key: CacheKey, dirPath: string): string[] { let paths = ''; if (!this.alwaysResetLdPath) { paths = process.env.LD_LIBRARY_PATH || ''; @@ -996,11 +1001,28 @@ export class BaseCompiler implements ICompiler { return _.union( paths.split(path.delimiter).filter(p => !!p), this.compiler.ldPath, + this.getExtraLdPaths(key), this.compiler.libPath, - this.getSharedLibraryPaths(libraries, dirPath), + this.getSharedLibraryPaths(key.libraries, dirPath), ); } + getExtraLdPaths(key: CacheKey): string[] { + let toolchainPath: any; + if (key.options) { + toolchainPath = getToolchainPathWithOptionsArr(this.compiler.exe, key.options) || this.toolchainPath; + } + + if (toolchainPath) { + const sysrootPath = getSysrootByToolchainPath(toolchainPath); + if (sysrootPath) { + return [path.join(sysrootPath, 'lib')]; + } + } + + return []; + } + getIncludeArguments(libraries: SelectedLibraryVersion[], dirPath: string): string[] { const includeFlag = this.compiler.includeFlag || '-I'; return libraries.flatMap(selectedLib => { @@ -1608,7 +1630,7 @@ export class BaseCompiler implements ICompiler { return utils.changeExtension(inputFilename, '.ll'); } - getOutputFilename(dirPath: string, outputFilebase: string, key?: CacheKey): string { + getOutputFilename(dirPath: string, outputFilebase: string, key?: any): string { let filename; if (key && key.backendOptions && key.backendOptions.customOutputFilename) { filename = key.backendOptions.customOutputFilename; @@ -1623,7 +1645,7 @@ export class BaseCompiler implements ICompiler { } } - getExecutableFilename(dirPath: string, outputFilebase: string, key?: CacheKey) { + getExecutableFilename(dirPath: string, outputFilebase: string, key?: CacheKey | CompilationCacheKey) { return this.getOutputFilename(dirPath, outputFilebase, key); } @@ -1888,44 +1910,45 @@ export class BaseCompiler implements ICompiler { return buildResult; } - async getOrBuildExecutable(key: CacheKey, bypassCache: BypassCache) { + async getOrBuildExecutable( + key: CacheKey, + bypassCache: BypassCache, + executablePackageHash: string, + ): Promise { const dirPath = await this.newTempDir(); if (!bypassCompilationCache(bypassCache)) { - const buildResults = await this.loadPackageWithExecutable(key, dirPath); - if (buildResults) return buildResults; + const buildResult = await this.loadPackageWithExecutable(key, executablePackageHash, dirPath); + if (buildResult) return buildResult; } - let compilationResult: BuildResult; + let buildResult: BuildResult; try { - compilationResult = await this.buildExecutableInFolder(key, dirPath); - if (compilationResult.code !== 0) { - return compilationResult; + buildResult = await this.buildExecutableInFolder(key, dirPath); + if (buildResult.code !== 0) { + return buildResult; } } catch (e) { - return this.handleUserError(e, dirPath); + return this.handleUserBuildError(e, dirPath); } - compilationResult.preparedLdPaths = _.union( - this.compiler.ldPath, - this.getSharedLibraryPathsAsLdLibraryPathsForExecution(key.libraries, dirPath), - ); - compilationResult.defaultExecOptions = this.getDefaultExecOptions(); + buildResult.preparedLdPaths = this.getSharedLibraryPathsAsLdLibraryPathsForExecution(key, dirPath); + buildResult.defaultExecOptions = this.getDefaultExecOptions(); - await this.storePackageWithExecutable(key, dirPath, compilationResult); + await this.storePackageWithExecutable(executablePackageHash, dirPath, buildResult); - if (!compilationResult.dirPath) { - compilationResult.dirPath = dirPath; + if (!buildResult.dirPath) { + buildResult.dirPath = dirPath; } - return compilationResult; + return buildResult; } - async loadPackageWithExecutable(key: CacheKey, dirPath: string) { + async loadPackageWithExecutable(key: CacheKey, executablePackageHash: string, dirPath: string) { const compilationResultFilename = 'compilation-result.json'; try { const startTime = process.hrtime.bigint(); - const outputFilename = await this.env.executableGet(key, dirPath); + const outputFilename = await this.env.executableGet(executablePackageHash, dirPath); if (outputFilename) { logger.debug(`Using cached package ${outputFilename}`); await this.packager.unpack(outputFilename, dirPath); @@ -1953,16 +1976,25 @@ export class BaseCompiler implements ICompiler { return false; } - async storePackageWithExecutable(key: CacheKey, dirPath: string, compilationResult: CompilationResult) { + async storePackageWithExecutable( + executablePackageHash: string, + dirPath: string, + compilationResult: CompilationResult, + ): Promise { const compilationResultFilename = 'compilation-result.json'; const packDir = await this.newTempDir(); const packagedFile = path.join(packDir, 'package.tgz'); try { - await fs.writeFile(path.join(dirPath, compilationResultFilename), JSON.stringify(compilationResult)); + // first remove tmpdir from executableFilename, this path will never be the same + // (it's kept in the original compilationResult to keep Tools from breaking that want the full path) + // note: couldn't use structuredClone() here, not sure why not + const clonedResult = JSON.parse(JSON.stringify(compilationResult)); + clonedResult.executableFilename = utils.maskRootdir(clonedResult.executableFilename); + + await fs.writeFile(path.join(dirPath, compilationResultFilename), JSON.stringify(clonedResult)); await this.packager.package(dirPath, packagedFile); - const hash = await this.env.executablePut(key, packagedFile); - logger.debug('Storing ' + hash); + await this.env.executablePut(executablePackageHash, packagedFile); } catch (err) { logger.error('Caught an error trying to put to cache: ', {err}); } finally { @@ -1974,6 +2006,15 @@ export class BaseCompiler implements ICompiler { return utils.processExecutionResult(input, inputFilename); } + async runExecutableRemotely( + executablePackageHash: string, + executeOptions: ExecutableExecutionOptions, + execTriple: BaseExecutionTriple, + ): Promise { + const env = new RemoteExecutionEnvironment(this.env, execTriple, executablePackageHash); + return await env.execute(executeOptions); + } + async runExecutable( executable: string, executeParameters: ExecutableExecutionOptions, @@ -2038,7 +2079,10 @@ export class BaseCompiler implements ICompiler { if (this.compiler.interpreted) { return this.handleInterpreting(key, executeParameters); } - const buildResult = await this.getOrBuildExecutable(key, bypassCache); + + const executablePackageHash = this.env.getExecutableHash(key); + + const buildResult = await this.getOrBuildExecutable(key, bypassCache, executablePackageHash); if (buildResult.code !== 0) { return { code: -1, @@ -2065,27 +2109,42 @@ export class BaseCompiler implements ICompiler { return verboseResult; } - if (!this.compiler.supportsExecute) { - return { - code: -1, - didExecute: false, - buildResult, - stderr: [{text: 'Compiler does not support execution'}], - stdout: [], - timedOut: false, - }; - } - if (buildResult.preparedLdPaths) { executeParameters.ldPath = buildResult.preparedLdPaths; } else { executeParameters.ldPath = this.getSharedLibraryPathsAsLdLibraryPathsForExecution( - key.libraries, - buildResult.dirPath, + key, + buildResult.dirPath || '', ); } - const result = await this.runExecutable(buildResult.executableFilename, executeParameters, buildResult.dirPath); + const execTriple = await RemoteExecutionQuery.guessExecutionTripleForBuildresult(buildResult); + if (!matchesCurrentHost(execTriple)) { + if (await RemoteExecutionQuery.isPossible(execTriple)) { + const result = await this.runExecutableRemotely(executablePackageHash, executeParameters, execTriple); + return moveArtifactsIntoResult(buildResult, { + ...result, + didExecute: true, + buildResult: buildResult, + }); + } else { + return { + code: -1, + didExecute: false, + buildResult, + stderr: [{text: `No execution available for ${execTriple.toString()}`}], + stdout: [], + execTime: '', + timedOut: false, + }; + } + } + + const result = await this.runExecutable( + buildResult.executableFilename, + executeParameters, + unwrap(buildResult.dirPath), + ); return moveArtifactsIntoResult(buildResult, { ...result, didExecute: true, @@ -2431,6 +2490,21 @@ export class BaseCompiler implements ICompiler { }; } + handleUserBuildError(error, dirPath): BuildResult { + return { + dirPath, + okToCache: false, + code: -1, + timedOut: false, + asm: [{text: `<${error.message}>`}], + stdout: [], + stderr: [{text: `<${error.message}>`}], + downloads: [], + executableFilename: '', + compilationOptions: [], + }; + } + async doBuildstepAndAddToResult( result: CompilationResult, name: string, @@ -2515,7 +2589,11 @@ export class BaseCompiler implements ICompiler { } } - async cmake(files: FiledataPair[], key: ParsedRequest, bypassCache: BypassCache): Promise { + async cmake( + files: FiledataPair[], + parsedRequest: ParsedRequest, + bypassCache: BypassCache, + ): Promise { // key = {source, options, backendOptions, filters, bypassCache, tools, executeParameters, libraries}; if (!this.compiler.supportsBinary) { @@ -2531,33 +2609,35 @@ export class BaseCompiler implements ICompiler { return errorResult; } - _.defaults(key.filters, this.getDefaultFilters()); - key.filters.binary = true; - key.filters.dontMaskFilenames = true; + _.defaults(parsedRequest.filters, this.getDefaultFilters()); + parsedRequest.filters.binary = true; + parsedRequest.filters.dontMaskFilenames = true; - const libsAndOptions = this.createLibsAndOptions(key); + const libsAndOptions = this.createLibsAndOptions(parsedRequest); - const toolchainPath = this.getDefaultOrOverridenToolchainPath(key.backendOptions.overrides || []); + const toolchainPath = this.getDefaultOrOverridenToolchainPath(parsedRequest.backendOptions.overrides || []); const dirPath = await this.newTempDir(); - const doExecute = key.filters.execute; + const doExecute = parsedRequest.filters.execute; + // todo: executeOptions.env should be set?? const executeOptions: ExecutableExecutionOptions = { - args: (key.executeParameters.args as string[]) || [], - stdin: key.executeParameters.stdin || '', - ldPath: this.getSharedLibraryPathsAsLdLibraryPaths(key.libraries, dirPath), - runtimeTools: key.executeParameters?.runtimeTools || [], + args: parsedRequest.executeParameters.args || [], + stdin: parsedRequest.executeParameters.stdin || '', + ldPath: this.getSharedLibraryPathsAsLdLibraryPaths(parsedRequest.libraries, dirPath), + runtimeTools: parsedRequest.executeParameters?.runtimeTools || [], env: {}, }; - const cacheKey = this.getCmakeCacheKey(key, files); + const cacheKey = this.getCmakeCacheKey(parsedRequest, files); + const executablePackageHash = this.env.getExecutableHash(cacheKey); const outputFilename = this.getExecutableFilename(path.join(dirPath, 'build'), this.outputFilebase, cacheKey); let fullResult: CompilationResult = bypassExecutionCache(bypassCache) ? null - : await this.loadPackageWithExecutable(cacheKey, dirPath); + : await this.loadPackageWithExecutable(cacheKey, executablePackageHash, dirPath); if (fullResult) { fullResult.retreivedFromCache = true; @@ -2591,12 +2671,15 @@ export class BaseCompiler implements ICompiler { fullResult.downloads = await this.setupBuildEnvironment(cacheKey, dirPath, true); - const toolchainparam = this.getCMakeExtToolchainParam(key.backendOptions.overrides || []); + const toolchainparam = this.getCMakeExtToolchainParam(parsedRequest.backendOptions.overrides || []); - const cmakeArgs = utils.splitArguments(key.backendOptions.cmakeArgs); - const partArgs: string[] = [toolchainparam, ...this.getExtraCMakeArgs(key), ...cmakeArgs, '..'].filter( - Boolean, - ); // filter out empty args + const cmakeArgs = utils.splitArguments(parsedRequest.backendOptions.cmakeArgs); + const partArgs: string[] = [ + toolchainparam, + ...this.getExtraCMakeArgs(parsedRequest), + ...cmakeArgs, + '..', + ].filter(Boolean); // filter out empty args const useNinja = this.env.ceProps('useninja'); const fullArgs: string[] = useNinja ? ['-GNinja'].concat(partArgs) : partArgs; @@ -2656,7 +2739,7 @@ export class BaseCompiler implements ICompiler { compilationOptions: this.getUsedEnvironmentVariableFlags(makeExecParams), }; - if (!key.backendOptions.skipAsm) { + if (!parsedRequest.backendOptions.skipAsm) { const [asmResult] = await this.checkOutputFileAndDoPostProcess( fullResult.result, outputFilename, @@ -2672,14 +2755,43 @@ export class BaseCompiler implements ICompiler { }); } - await this.storePackageWithExecutable(cacheKey, dirPath, fullResult); + await this.storePackageWithExecutable(executablePackageHash, dirPath, fullResult); } - if (fullResult.result) fullResult.result.dirPath = dirPath; + if (fullResult.result) { + fullResult.result.dirPath = dirPath; - if (this.compiler.supportsExecute && doExecute) { - fullResult.execResult = await this.runExecutable(outputFilename, executeOptions, dirPath); - fullResult.didExecute = true; + if (doExecute && fullResult.result.code === 0) { + const execTriple = await RemoteExecutionQuery.guessExecutionTripleForBuildresult({ + ...fullResult, + downloads: fullResult.downloads || [], + executableFilename: outputFilename, + compilationOptions: fullResult.compilationOptions || [], + }); + + if (matchesCurrentHost(execTriple)) { + fullResult.execResult = await this.runExecutable(outputFilename, executeOptions, dirPath); + fullResult.didExecute = true; + } else { + if (await RemoteExecutionQuery.isPossible(execTriple)) { + fullResult.execResult = await this.runExecutableRemotely( + executablePackageHash, + executeOptions, + execTriple, + ); + fullResult.didExecute = true; + } else { + fullResult.execResult = { + code: -1, + okToCache: false, + stdout: [], + stderr: [{text: `No execution available for ${execTriple.toString()}`}], + execTime: '', + timedOut: false, + }; + } + } + } } const optOutput = undefined; @@ -2689,7 +2801,7 @@ export class BaseCompiler implements ICompiler { false, cacheKey, executeOptions, - key.tools, + parsedRequest.tools, cacheKey.backendOptions, cacheKey.filters, libsAndOptions.options, diff --git a/lib/binaries/binary-utils.ts b/lib/binaries/binary-utils.ts new file mode 100644 index 000000000..c1d7db7fe --- /dev/null +++ b/lib/binaries/binary-utils.ts @@ -0,0 +1,128 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import {InstructionSet} from '../../types/instructionsets.js'; +import {executeDirect} from '../exec.js'; +import {logger} from '../logger.js'; + +export enum OSType { + linux = 'linux', + windows = 'win32', +} + +export type BinaryInfo = { + instructionSet: InstructionSet; + os: OSType; +}; + +export class BinaryInfoLinux { + static getInstructionSetForArchText(value: string): InstructionSet { + switch (value) { + case 'x86-64': { + return 'amd64'; + } + case 'intel 80386': { + return 'x86'; + } + case '80386': { + return 'x86'; + } + case 'arm aarch64': { + return 'aarch64'; + } + case 'aarch64': { + return 'aarch64'; + } + case 'arm': { + return 'arm32'; + } + case 'atmel avr 8-bit': { + return 'avr'; + } + case 'ucb risc-v': { + return 'riscv64'; + } + case '64-bit powerpc or cisco 7500': { + return 'powerpc'; + } + case 'powerpc or cisco 4500': { + return 'powerpc'; + } + case 'mips': { + return 'mips'; + } + case 'loongarch': { + return 'loongarch'; + } + default: { + logger.error(`Unknown architecture text: ${value}`); + return 'amd64'; + } + } + } + + static removeComments(value: string): string { + let filtered: string = ''; + let inComment: boolean = false; + for (const c of value) { + if (!inComment && c === '(') { + inComment = true; + } else if (inComment && c === ')') { + inComment = false; + } else if (!inComment) { + filtered += c; + } + } + return filtered.trim(); + } + + static parseFileInfo(output: string): BinaryInfo | undefined { + const csv: string[] = output.split(', ').map(val => val.trim().toLowerCase()); + const isElf = csv[0].startsWith('elf'); + const isPE = csv[0].startsWith('pe32'); + if (isElf) { + return { + os: OSType.linux, + instructionSet: this.getInstructionSetForArchText(csv[1]), + }; + } else if (isPE) { + const filteredLine = this.removeComments(csv[0]); + const lastWordPos = filteredLine.lastIndexOf(' '); + const lastWord = filteredLine.substring(lastWordPos + 1); + + return { + os: OSType.windows, + instructionSet: this.getInstructionSetForArchText(lastWord), + }; + } + + return undefined; + } + + static async readFile(filepath: string): Promise { + const info = await executeDirect('/usr/bin/file', ['-b', filepath], {}); + if (info.code === 0) return this.parseFileInfo(info.stdout); + return undefined; + } +} diff --git a/lib/win-utils.ts b/lib/binaries/win-utils.ts similarity index 95% rename from lib/win-utils.ts rename to lib/binaries/win-utils.ts index 49dbeee70..69600716a 100644 --- a/lib/win-utils.ts +++ b/lib/binaries/win-utils.ts @@ -26,11 +26,10 @@ import path from 'path'; import * as fs from 'fs-extra'; -import {ExecutionOptionsWithEnv} from '../types/compilation/compilation.interfaces.js'; - -import {BaseCompiler} from './base-compiler.js'; -import {logger} from './logger.js'; -import * as utils from './utils.js'; +import {ExecutionOptionsWithEnv} from '../../types/compilation/compilation.interfaces.js'; +import {BaseCompiler} from '../base-compiler.js'; +import {logger} from '../logger.js'; +import * as utils from '../utils.js'; export class WinUtils { protected re_dll_name = /dll name: (.*\.dll)/i; diff --git a/lib/compilation-env.ts b/lib/compilation-env.ts index 7999dcd61..c885b7e1b 100644 --- a/lib/compilation-env.ts +++ b/lib/compilation-env.ts @@ -38,13 +38,12 @@ import {CompilationQueue, EnqueueOptions, Job} from './compilation-queue.js'; import {FormattingHandler} from './handlers/formatting.js'; import {logger} from './logger.js'; import type {PropertyGetter} from './properties.interfaces.js'; -import {CompilerProps} from './properties.js'; +import {CompilerProps, PropFunc} from './properties.js'; import {createStatsNoter, IStatsNoter} from './stats.js'; -type PropFunc = (s: string, a?: any) => any; - export class CompilationEnvironment { ceProps: PropertyGetter; + awsProps: PropFunc; compilationQueue: CompilationQueue | undefined; compilerProps: PropFunc; okOptions: RegExp; @@ -60,8 +59,14 @@ export class CompilationEnvironment { statsNoter: IStatsNoter; private logCompilerCacheAccesses: boolean; - constructor(compilerProps: CompilerProps, compilationQueue: CompilationQueue | undefined, doCache?: boolean) { + constructor( + compilerProps: CompilerProps, + awsProps: PropFunc, + compilationQueue: CompilationQueue | undefined, + doCache?: boolean, + ) { this.ceProps = compilerProps.ceProps; + this.awsProps = awsProps; this.compilationQueue = compilationQueue; this.compilerProps = compilerProps.get.bind(compilerProps); // So people running local instances don't break suddenly when updating @@ -158,8 +163,11 @@ export class CompilationEnvironment { return this.compilerCache.put(key, JSON.stringify(result), creator); } - async executableGet(object: CacheableValue, destinationFolder: string): Promise { - const key = BaseCache.hash(object) + '_exec'; + getExecutableHash(object: CacheableValue): string { + return BaseCache.hash(object) + '_exec'; + } + + async executableGet(key: string, destinationFolder: string): Promise { const result = await this.executableCache.get(key); if (!result.hit) return null; const filepath = destinationFolder + '/' + key; @@ -167,10 +175,8 @@ export class CompilationEnvironment { return filepath; } - async executablePut(object: CacheableValue, filepath: string): Promise { - const key = BaseCache.hash(object) + '_exec'; + async executablePut(key: string, filepath: string): Promise { await this.executableCache.put(key, fs.readFileSync(filepath)); - return key; } enqueue(job: Job, options?: EnqueueOptions) { diff --git a/lib/compilers/cerberus.ts b/lib/compilers/cerberus.ts index 6d9a51213..94ef8a682 100644 --- a/lib/compilers/cerberus.ts +++ b/lib/compilers/cerberus.ts @@ -29,6 +29,7 @@ import {BypassCache, CacheKey, ExecutionOptions} from '../../types/compilation/c import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; import {ExecutableExecutionOptions} from '../../types/execution/execution.interfaces.js'; import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; +import {assert} from '../assert.js'; import {BaseCompiler} from '../base-compiler.js'; import {CompilationEnvironment} from '../compilation-env.js'; import {logger} from '../logger.js'; @@ -89,7 +90,9 @@ export class CerberusCompiler extends BaseCompiler { } override async handleInterpreting(key: CacheKey, executeParameters: ExecutableExecutionOptions) { - const compileResult = await this.getOrBuildExecutable(key, BypassCache.None); + const executionPackageHash = this.env.getExecutableHash(key); + const compileResult = await this.getOrBuildExecutable(key, BypassCache.None, executionPackageHash); + assert(compileResult.dirPath !== undefined); if (compileResult.code === 0) { executeParameters.args = [ '--exec', diff --git a/lib/compilers/java.ts b/lib/compilers/java.ts index 92186a93b..5501b9e1c 100644 --- a/lib/compilers/java.ts +++ b/lib/compilers/java.ts @@ -32,7 +32,7 @@ import {BypassCache, CacheKey, CompilationResult} from '../../types/compilation/ import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; import {ExecutableExecutionOptions} from '../../types/execution/execution.interfaces.js'; import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; -import {unwrap} from '../assert.js'; +import {assert, unwrap} from '../assert.js'; import {BaseCompiler, SimpleOutputFilenameCompiler} from '../base-compiler.js'; import {CompilationEnvironment} from '../compilation-env.js'; import {logger} from '../logger.js'; @@ -135,12 +135,14 @@ export class JavaCompiler extends BaseCompiler implements SimpleOutputFilenameCo key: CacheKey, executeParameters: ExecutableExecutionOptions, ): Promise { - const compileResult = await this.getOrBuildExecutable(key, BypassCache.None); + const executionPackageHash = this.env.getExecutableHash(key); + const compileResult = await this.getOrBuildExecutable(key, BypassCache.None, executionPackageHash); if (compileResult.code === 0) { const extraXXFlags: string[] = []; if (Semver.gte(utils.asSafeVer(this.compiler.semver), '11.0.0', true)) { extraXXFlags.push('-XX:-UseDynamicNumberOfCompilerThreads'); } + assert(compileResult.dirPath !== undefined); executeParameters.args = [ '-Xss136K', // Reduce thread stack size '-XX:CICompilerCount=2', // Reduce JIT compilation threads. 2 is minimum diff --git a/lib/compilers/kotlin.ts b/lib/compilers/kotlin.ts index e8a090208..e86bbb16d 100644 --- a/lib/compilers/kotlin.ts +++ b/lib/compilers/kotlin.ts @@ -26,6 +26,7 @@ import {BypassCache, CacheKey, CompilationResult} from '../../types/compilation/ import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; import {ExecutableExecutionOptions} from '../../types/execution/execution.interfaces.js'; import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; +import {assert} from '../assert.js'; import {SimpleOutputFilenameCompiler} from '../base-compiler.js'; import {CompilationEnvironment} from '../compilation-env.js'; @@ -105,7 +106,9 @@ export class KotlinCompiler extends JavaCompiler implements SimpleOutputFilename ...key, options: ['-include-runtime', '-d', 'example.jar'], }; - const compileResult = await this.getOrBuildExecutable(alteredKey, BypassCache.None); + const executablePackageHash = this.env.getExecutableHash(key); + const compileResult = await this.getOrBuildExecutable(alteredKey, BypassCache.None, executablePackageHash); + assert(compileResult.dirPath !== undefined); if (compileResult.code !== 0) { return { stdout: compileResult.stdout, diff --git a/lib/compilers/rust.ts b/lib/compilers/rust.ts index f1019c701..4e42af737 100644 --- a/lib/compilers/rust.ts +++ b/lib/compilers/rust.ts @@ -43,7 +43,8 @@ import {changeExtension, parseRustOutput} from '../utils.js'; import {RustParser} from './argument-parsers.js'; export class RustCompiler extends BaseCompiler { - linker: string; + amd64linker: string; + aarch64linker: string; static get key() { return 'rust'; @@ -69,7 +70,8 @@ export class RustCompiler extends BaseCompiler { moduleScopeArg: ['-C', 'llvm-args=-print-module-scope'], noDiscardValueNamesArg: isNightly ? ['-Z', 'fewer-names=no'] : [], }; - this.linker = this.compilerProps('linker'); + this.amd64linker = this.compilerProps('linker'); + this.aarch64linker = this.compilerProps('aarch64linker'); } override async generateIR( @@ -207,6 +209,26 @@ export class RustCompiler extends BaseCompiler { return [options, overrides]; } + override changeOptionsBasedOnOverrides(options: string[], overrides: ConfiguredOverrides): string[] { + const newOptions = super.changeOptionsBasedOnOverrides(options, overrides); + + if (this.aarch64linker) { + const useAarch64Linker = !!overrides.find( + override => override.name === CompilerOverrideType.arch && override.value.includes('aarch64'), + ); + + if (useAarch64Linker) { + for (let idx = 0; idx < newOptions.length; idx++) { + if (newOptions[idx].indexOf('-Clinker=') === 0) { + newOptions[idx] = `-Clinker=${this.aarch64linker}`; + } + } + } + } + + return newOptions; + } + override optionsForBackend(backendOptions: Record, outputFilename: string) { // The super class handles the GCC dump files that may be needed by // rustc-cg-gcc subclass. @@ -225,8 +247,8 @@ export class RustCompiler extends BaseCompiler { const userRequestedEmit = _.any(unwrap(userOptions), opt => opt.includes('--emit')); if (filters.binary) { options = options.concat(['--crate-type', 'bin']); - if (this.linker) { - options = options.concat(`-Clinker=${this.linker}`); + if (this.amd64linker) { + options = options.concat(`-Clinker=${this.amd64linker}`); } } else if (filters.binaryObject) { options = options.concat(['--crate-type', 'lib']); diff --git a/lib/compilers/win32-mingw-clang.ts b/lib/compilers/win32-mingw-clang.ts index abfc4068e..6be35bb14 100644 --- a/lib/compilers/win32-mingw-clang.ts +++ b/lib/compilers/win32-mingw-clang.ts @@ -27,7 +27,7 @@ import path from 'path'; import {BuildResult, BypassCache, CacheKey, CompilationResult} from '../../types/compilation/compilation.interfaces.js'; import {ExecutableExecutionOptions} from '../../types/execution/execution.interfaces.js'; import {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; -import {copyNeededDlls} from '../win-utils.js'; +import {copyNeededDlls} from '../binaries/win-utils.js'; import {ClangCompiler} from './clang.js'; diff --git a/lib/compilers/win32-mingw-gcc.ts b/lib/compilers/win32-mingw-gcc.ts index 865137e08..01788e33f 100644 --- a/lib/compilers/win32-mingw-gcc.ts +++ b/lib/compilers/win32-mingw-gcc.ts @@ -27,7 +27,7 @@ import path from 'path'; import {BuildResult, BypassCache, CacheKey, CompilationResult} from '../../types/compilation/compilation.interfaces.js'; import {ExecutableExecutionOptions} from '../../types/execution/execution.interfaces.js'; import {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; -import {copyNeededDlls} from '../win-utils.js'; +import {copyNeededDlls} from '../binaries/win-utils.js'; import {GCCCompiler} from './gcc.js'; diff --git a/lib/execution/base-execution-env.ts b/lib/execution/base-execution-env.ts index 85a29b3ab..fb70cff7b 100644 --- a/lib/execution/base-execution-env.ts +++ b/lib/execution/base-execution-env.ts @@ -144,8 +144,6 @@ export class LocalExecutionEnvironment implements IExecutionEnvironment { } } - // todo: what to do about the rest of the runtimeTools? - if ( this.buildResult && this.buildResult.defaultExecOptions && @@ -157,12 +155,20 @@ export class LocalExecutionEnvironment implements IExecutionEnvironment { else env.PATH = this.buildResult.defaultExecOptions.env.PATH; } + let extraLdPaths: string[] = []; + if (env.LD_LIBRARY_PATH) { + extraLdPaths = env.LD_LIBRARY_PATH.split(path.delimiter); + delete env.LD_LIBRARY_PATH; + } + const execOptions: ExecutionOptionsWithEnv = { env, }; if (this.buildResult && this.buildResult.preparedLdPaths) { - execOptions.ldPath = this.buildResult.preparedLdPaths; + execOptions.ldPath = this.buildResult.preparedLdPaths.concat(extraLdPaths); + } else { + execOptions.ldPath = extraLdPaths; } return execOptions; @@ -170,6 +176,7 @@ export class LocalExecutionEnvironment implements IExecutionEnvironment { async execute(params: ExecutionParams): Promise { assert(this.buildResult); + assert(this.dirPath !== 'not initialized'); const execExecutableOptions: ExecutableExecutionOptions = { args: typeof params.args === 'string' ? utils.splitArguments(params.args) : params.args || [], @@ -179,15 +186,13 @@ export class LocalExecutionEnvironment implements IExecutionEnvironment { runtimeTools: params.runtimeTools, }; - const homeDir = await temp.mkdir({prefix: utils.ce_temp_prefix, dir: os.tmpdir()}); + // note: this is for a small transition period only, can be removed after a few days + const file = utils.maskRootdir(this.buildResult.executableFilename); - return await this.execBinary(this.buildResult.executableFilename, execExecutableOptions, homeDir); + return await this.execBinary(file, execExecutableOptions, this.dirPath); } - protected setEnvironmentVariablesFromRuntime( - configuredTools: ConfiguredRuntimeTools, - execOptions: ExecutionOptions, - ) { + static setEnvironmentVariablesFromRuntime(configuredTools: ConfiguredRuntimeTools, execOptions: ExecutionOptions) { for (const runtime of configuredTools) { if (runtime.name === RuntimeToolType.env) { for (const env of runtime.options) { @@ -264,7 +269,7 @@ export class LocalExecutionEnvironment implements IExecutionEnvironment { if (!execOptions.env) execOptions.env = {}; if (executeParameters.runtimeTools) { - this.setEnvironmentVariablesFromRuntime(executeParameters.runtimeTools, execOptions); + LocalExecutionEnvironment.setEnvironmentVariablesFromRuntime(executeParameters.runtimeTools, execOptions); for (const runtime of executeParameters.runtimeTools) { if (runtime.name === RuntimeToolType.heaptrack) { diff --git a/lib/execution/base-execution-triple.ts b/lib/execution/base-execution-triple.ts new file mode 100644 index 000000000..40d7a320d --- /dev/null +++ b/lib/execution/base-execution-triple.ts @@ -0,0 +1,80 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import {InstructionSet} from '../../types/instructionsets.js'; +import {OSType} from '../binaries/binary-utils.js'; + +export enum ExecutionSpecialty { + cpu = 'cpu', + nvgpu = 'nvgpu', + amdgpu = 'amdgpu', +} + +export class BaseExecutionTriple { + protected _instructionSet: InstructionSet = 'amd64'; + protected _os: OSType = OSType.linux; + protected _specialty: ExecutionSpecialty = ExecutionSpecialty.cpu; + + get instructionSet(): InstructionSet { + return this._instructionSet; + } + + set instructionSet(value: InstructionSet) { + this._instructionSet = value; + } + + set os(value: OSType) { + this._os = value; + } + + get os(): OSType { + return this._os; + } + + set specialty(value: ExecutionSpecialty) { + this._specialty = value; + } + + get specialty(): ExecutionSpecialty { + return this._specialty; + } + + toString(): string { + return `${this._instructionSet}-${this._os}-${this._specialty}`; + } + + parse(triple: string) { + if (triple.includes('-')) { + const reTriple = /(\w*)-(\w*)-(\w*)/; + const match = triple.match(reTriple); + if (match) { + this._instructionSet = match[1] as InstructionSet; + this._os = match[2] as OSType; + this._specialty = match[3] as ExecutionSpecialty; + } + } else { + this._instructionSet = triple as InstructionSet; + } + } +} diff --git a/lib/execution/events-websocket.ts b/lib/execution/events-websocket.ts new file mode 100644 index 000000000..f9c812ada --- /dev/null +++ b/lib/execution/events-websocket.ts @@ -0,0 +1,134 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import {WebSocket} from 'ws'; + +import {BasicExecutionResult} from '../../types/execution/execution.interfaces.js'; +import {logger} from '../logger.js'; +import {PropertyGetter} from '../properties.interfaces.js'; + +export class EventsWsBase { + protected expectClose: boolean = false; + protected events_url: string; + protected ws: WebSocket | undefined = undefined; + protected got_error: boolean = false; + + constructor(props: PropertyGetter) { + this.events_url = props('execqueue.events_url', ''); + if (this.events_url === '') throw new Error('execqueue.events_url property required'); + } + + protected connect() { + if (!this.ws) { + this.ws = new WebSocket(this.events_url); + this.ws.on('error', e => { + this.got_error = true; + logger.error(`Error while trying to communicate with websocket at URL ${this.events_url}`); + logger.error(e); + }); + } + } + + async close(): Promise { + this.expectClose = true; + if (this.ws) { + this.ws.close(); + } + } +} + +export class EventsWsSender extends EventsWsBase { + async send(guid: string, result: BasicExecutionResult): Promise { + this.connect(); + return new Promise(resolve => { + this.ws.on('open', async () => { + this.ws.send( + JSON.stringify({ + guid: guid, + ...result, + }), + ); + resolve(); + }); + }); + } +} + +export class EventsWsWaiter extends EventsWsBase { + private timeout: number; + + constructor(props: PropertyGetter) { + super(props); + + // binaryExecTimeoutMs + 2500 to allow for some generous network latency between completion and receiving the result + this.timeout = props('binaryExecTimeoutMs', 10000) + 2500; + } + + async subscribe(guid: string): Promise { + this.connect(); + return new Promise((resolve, reject) => { + const errorCheck = setInterval(() => { + if (this.got_error) { + reject(); + } + }, 500); + + this.ws.on('open', async () => { + this.ws.send(`subscribe: ${guid}`); + clearInterval(errorCheck); + resolve(); + }); + }); + } + + async data(): Promise { + let runningTime = 0; + return new Promise((resolve, reject) => { + const t = setInterval(() => { + runningTime = runningTime + 1000; + if (runningTime > this.timeout) { + clearInterval(t); + reject('Remote execution timed out without returning a result'); + } + }, 1000); + + this.ws.on('message', async message => { + clearInterval(t); + try { + const data = JSON.parse(message.toString()); + resolve(data); + } catch (e) { + reject(e); + } + }); + + this.ws.on('close', () => { + clearInterval(t); + if (!this.expectClose) { + reject('Unable to complete remote execution due to unexpected situation'); + } + }); + }); + } +} diff --git a/lib/execution/execution-query.ts b/lib/execution/execution-query.ts new file mode 100644 index 000000000..7d074da5c --- /dev/null +++ b/lib/execution/execution-query.ts @@ -0,0 +1,95 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import {BuildResult} from '../../types/compilation/compilation.interfaces.js'; +import {BinaryInfoLinux} from '../binaries/binary-utils.js'; +import {logger} from '../logger.js'; +import {PropertyGetter} from '../properties.interfaces.js'; + +import {BaseExecutionTriple, ExecutionSpecialty} from './base-execution-triple.js'; + +async function retrieveAllRemoteExecutionArchs(ceProps: PropertyGetter, envs: string[]): Promise { + let url: string = ceProps('execqueue.remote_archs_url', ''); + try { + if (!url) return []; + + if (envs.find(env => env.includes('staging') || env === 'dev')) url += '?env=staging'; + + // eslint-disable-next-line n/no-unsupported-features/node-builtins + const response = await fetch(url); + + if (response.status !== 200) { + throw new Error(`Status ${response.status} on GET ${url}`); + } + + return await response.json(); + } catch (e) { + logger.error('Error retrieving remote execution architectures'); + logger.error(e); + process.exit(1); + } +} + +let _available_remote_execution_archs: Promise; + +export class RemoteExecutionQuery { + static initRemoteExecutionArchs(ceProps: PropertyGetter, envs: string[]) { + _available_remote_execution_archs = new Promise(resolve => { + retrieveAllRemoteExecutionArchs(ceProps, envs).then(archs => { + const triples: BaseExecutionTriple[] = []; + for (const arch of archs) { + const triple = new BaseExecutionTriple(); + triple.parse(arch); + triples.push(triple); + } + resolve(triples); + }); + }); + } + + static async isPossible(triple: BaseExecutionTriple): Promise { + const triples = await _available_remote_execution_archs; + return !!triples.find(remote => remote.toString() === triple.toString()); + } + + static async guessExecutionTripleForBuildresult(result: BuildResult): Promise { + const triple = new BaseExecutionTriple(); + + if (result.executableFilename) { + const info = await BinaryInfoLinux.readFile(result.executableFilename); + if (info) { + triple.instructionSet = info.instructionSet; + triple.os = info.os; + } + } else { + if (result.instructionSet) triple.instructionSet = result.instructionSet; + } + + if (result.devices && Object.keys(result.devices).length > 0) { + triple.specialty = ExecutionSpecialty.nvgpu; + } + + return triple; + } +} diff --git a/lib/execution/execution-triple.ts b/lib/execution/execution-triple.ts new file mode 100644 index 000000000..7e0bd6ed0 --- /dev/null +++ b/lib/execution/execution-triple.ts @@ -0,0 +1,131 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import os from 'os'; + +import {InstructionSet} from '../../types/instructionsets.js'; +import {OSType} from '../binaries/binary-utils.js'; +import {logger} from '../logger.js'; +import * as utils from '../utils.js'; + +import {BaseExecutionTriple, ExecutionSpecialty} from './base-execution-triple.js'; + +// import fs from 'fs-extra'; + +let _host_specialty = ExecutionSpecialty.cpu; + +function setHostSpecialty(value: ExecutionSpecialty) { + _host_specialty = value; +} + +export async function initHostSpecialties(): Promise { + if (os.platform() !== 'win32') { + const nvidiaGpuExists = await utils.fileExists('/dev/nvidia0'); + if (nvidiaGpuExists) { + setHostSpecialty(ExecutionSpecialty.nvgpu); + } + + // const cpuInfo = await fs.readFile('/proc/cpuinfo'); + // if (cpuInfo.includes('GenuineIntel')) { + + // } + } +} + +class CurrentHostExecHelper { + static isetCanRunOnCurrentHost(value: InstructionSet): boolean { + // os.arch() Possible values are `'arm'`, `'arm64'`, `'ia32'`, `'loong64'`,`'mips'`, `'mipsel'`, `'ppc'`, `'ppc64'`, `'riscv64'`, `'s390'`, `'s390x'`, and `'x64'`. + + const hostArch = os.arch(); + if (hostArch === 'arm64' && value === 'aarch64') { + return true; + } else if (hostArch === 'arm' && value === 'arm32') { + return true; + } else if (hostArch === 'x64' && (value === 'amd64' || value === 'x86')) { + return true; + } else if (hostArch === 'ia32' && value === 'x86') { + return true; + } + + return false; + } + + static osMatchesCurrentHost(value: string): boolean { + // Possible values are `'aix'`, `'darwin'`, `'freebsd'`, `'linux'`, `'openbsd'`, `'sunos'`, and `'win32'` + return value === os.platform(); + } + + static specialtyMatchesCurrentHost(value: ExecutionSpecialty) { + if (value === _host_specialty) return true; + + if (_host_specialty === ExecutionSpecialty.nvgpu && value === ExecutionSpecialty.cpu) return true; + + return _host_specialty === ExecutionSpecialty.amdgpu && value === ExecutionSpecialty.cpu; + } + + static getInstructionSetByNodeJSArch(value: string): InstructionSet { + switch (value) { + case 'x64': { + return 'amd64'; + } + case 'ia32': { + return 'x86'; + } + case 'arm64': { + return 'aarch64'; + } + case 'arm': { + return 'arm32'; + } + default: { + return os.arch() as InstructionSet; + } + } + } +} + +// note: returns array of BaseExecutionTriple to prepare for the future of fulfilling multiple execution roles, not actually implemented +export function getExecutionTriplesForCurrentHost(): BaseExecutionTriple[] { + const triple = new BaseExecutionTriple(); + triple.instructionSet = CurrentHostExecHelper.getInstructionSetByNodeJSArch(os.arch()); + + const platform = os.platform() as string; + if ((Object.values(OSType) as string[]).includes(platform)) { + triple.os = platform as OSType; + } else { + logger.warning(`getExecutionTripleForCurrentHost - Unsupported platform ${platform}`); + } + + triple.specialty = _host_specialty; + + return [triple]; +} + +export function matchesCurrentHost(triple: BaseExecutionTriple): boolean { + const matchesArch = CurrentHostExecHelper.isetCanRunOnCurrentHost(triple.instructionSet); + const matchesOS = CurrentHostExecHelper.osMatchesCurrentHost(triple.os); + const matchesSpecialty = CurrentHostExecHelper.specialtyMatchesCurrentHost(triple.specialty); + + return matchesArch && matchesOS && matchesSpecialty; +} diff --git a/lib/execution/remote-execution-env.ts b/lib/execution/remote-execution-env.ts new file mode 100644 index 000000000..5a0296b0f --- /dev/null +++ b/lib/execution/remote-execution-env.ts @@ -0,0 +1,110 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import crypto from 'crypto'; + +import {ExecutionParams} from '../../types/compilation/compilation.interfaces.js'; +import {BasicExecutionResult, ExecutableExecutionOptions} from '../../types/execution/execution.interfaces.js'; +import {CompilationEnvironment} from '../compilation-env.js'; +import {logger} from '../logger.js'; + +import {BaseExecutionTriple} from './base-execution-triple.js'; +import {EventsWsWaiter} from './events-websocket.js'; +import {IExecutionEnvironment} from './execution-env.interfaces.js'; +import {SqsExecuteRequester} from './sqs-execution-queue.js'; + +export class RemoteExecutionEnvironment implements IExecutionEnvironment { + private packageHash: string; + private triple: BaseExecutionTriple; + private execQueue: SqsExecuteRequester; + private guid: string; + private environment: CompilationEnvironment; + + constructor(environment: CompilationEnvironment, triple: BaseExecutionTriple, executablePackageHash: string) { + this.environment = environment; + this.triple = triple; + this.guid = crypto.randomUUID(); + this.packageHash = executablePackageHash; + this.execQueue = new SqsExecuteRequester(environment.ceProps, environment.awsProps); + + logger.info( + `RemoteExecutionEnvironment with ${triple.toString()} and ${executablePackageHash} - guid ${this.guid}`, + ); + } + + async downloadExecutablePackage(hash: string): Promise { + throw new Error('Method not implemented.'); + } + + private async queueRemoteExecution(params: ExecutionParams) { + await this.execQueue.push(this.triple, { + guid: this.guid, + hash: this.packageHash, + params: params, + }); + } + + async execute(params: ExecutionParams): Promise { + const startTime = process.hrtime.bigint(); + const waiter = new EventsWsWaiter(this.environment.ceProps); + try { + await waiter.subscribe(this.guid); + + await this.queueRemoteExecution(params); + + const result = await waiter.data(); + + await waiter.close(); + + const endTime = process.hrtime.bigint(); + + // change time to include overhead like SQS, WS, network etc + result.processExecutionResultTime = + parseInt((endTime - startTime).toString()) / 1000000 - parseInt(result.execTime); + + return result; + } catch (e) { + waiter.close(); + + return { + code: -1, + stdout: [], + stderr: [{text: 'Internal error while trying to remotely execute'}], + timedOut: false, + execTime: '', + okToCache: false, + filenameTransform: f => f, + }; + } + } + + async execBinary( + executable: string, + executeParameters: ExecutableExecutionOptions, + homeDir: string, + extraConfiguration?: any, + ): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/lib/execution/sqs-execution-queue.ts b/lib/execution/sqs-execution-queue.ts new file mode 100644 index 000000000..0957517a4 --- /dev/null +++ b/lib/execution/sqs-execution-queue.ts @@ -0,0 +1,185 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import {SQS} from '@aws-sdk/client-sqs'; + +import {ExecutionParams} from '../../types/compilation/compilation.interfaces.js'; +import {BasicExecutionResult} from '../../types/execution/execution.interfaces.js'; +import {logger} from '../logger.js'; +import {PropertyGetter} from '../properties.interfaces.js'; +import {getHash} from '../utils.js'; + +import {LocalExecutionEnvironment} from './_all.js'; +import {BaseExecutionTriple} from './base-execution-triple.js'; +import {EventsWsSender} from './events-websocket.js'; +import {getExecutionTriplesForCurrentHost} from './execution-triple.js'; + +export type RemoteExecutionMessage = { + guid: string; + hash: string; + params: ExecutionParams; +}; + +export class SqsExecuteQueueBase { + protected sqs: SQS; + protected queue_url: string; + + constructor(props: PropertyGetter, awsProps: PropertyGetter) { + const region = awsProps('region', ''); + this.sqs = new SQS({region: region}); + this.queue_url = props('execqueue.queue_url', ''); + if (this.queue_url === '') throw new Error('execqueue.queue_url property required'); + } + + getSqsQueueUrl(triple: BaseExecutionTriple) { + return this.queue_url + '-' + triple.toString() + '.fifo'; + } +} + +export class SqsExecuteRequester extends SqsExecuteQueueBase { + private async sendMsg(url: string, body: string) { + try { + return await this.sqs.sendMessage({ + QueueUrl: url, + MessageBody: body, + MessageGroupId: 'default', + MessageDeduplicationId: getHash(body), + }); + } catch (e) { + logger.error(`Error sending message to queue with URL: ${url}`); + throw e; + } + } + + async push(triple: BaseExecutionTriple, message: RemoteExecutionMessage): Promise { + const body = JSON.stringify(message); + const url = this.getSqsQueueUrl(triple); + + return this.sendMsg(url, body); + } +} + +export class SqsWorkerMode extends SqsExecuteQueueBase { + protected triples: BaseExecutionTriple[]; + + constructor(props: PropertyGetter, awsProps: PropertyGetter) { + super(props, awsProps); + this.triples = getExecutionTriplesForCurrentHost(); + } + + private async receiveMsg(url) { + try { + return await this.sqs.receiveMessage({ + QueueUrl: url, + MaxNumberOfMessages: 1, + }); + } catch (e) { + logger.error(`Error retreiving message from queue with URL: ${url}`); + throw e; + } + } + + async pop(): Promise { + const url = this.getSqsQueueUrl(this.triples[0]); + + const queued_messages = await this.receiveMsg(url); + + if (queued_messages.Messages && queued_messages.Messages.length === 1) { + const queued_message = queued_messages.Messages[0]; + + try { + if (queued_message.Body) { + const json = queued_message.Body; + return JSON.parse(json) as RemoteExecutionMessage; + } else { + return undefined; + } + } finally { + if (queued_message.ReceiptHandle) { + await this.sqs.deleteMessage({ + QueueUrl: url, + ReceiptHandle: queued_message.ReceiptHandle, + }); + } + } + } + + return undefined; + } +} + +async function sendResultViaWebsocket(compilationEnvironment, guid: string, result: BasicExecutionResult) { + try { + const sender = new EventsWsSender(compilationEnvironment.ceProps); + await sender.send(guid, result); + await sender.close(); + } catch (error) { + logger.error(error); + } +} + +async function doOneExecution(queue, compilationEnvironment) { + const msg = await queue.pop(); + if (msg && msg.guid) { + try { + const executor = new LocalExecutionEnvironment(compilationEnvironment); + await executor.downloadExecutablePackage(msg.hash); + const result = await executor.execute(msg.params); + + await sendResultViaWebsocket(compilationEnvironment, msg.guid, result); + } catch (e) { + // todo: e is undefined somehow? + logger.error(e); + + await sendResultViaWebsocket(compilationEnvironment, msg.guid, { + code: -1, + stderr: [{text: 'Internal error when remotely executing'}], + stdout: [], + okToCache: false, + timedOut: false, + filenameTransform: f => f, + execTime: '0', + }); + } + } +} + +export function startExecutionWorkerThread(ceProps, awsProps, compilationEnvironment) { + const queue = new SqsWorkerMode(ceProps, awsProps); + + // allow 2 executions at the same time + + const doExecutionWork1 = async () => { + await doOneExecution(queue, compilationEnvironment); + setTimeout(doExecutionWork1, 100); + }; + + const doExecutionWork2 = async () => { + await doOneExecution(queue, compilationEnvironment); + setTimeout(doExecutionWork2, 100); + }; + + setTimeout(doExecutionWork1, 1500); + setTimeout(doExecutionWork2, 1530); +} diff --git a/lib/handlers/compile.interfaces.ts b/lib/handlers/compile.interfaces.ts index ec214dcab..3888c06af 100644 --- a/lib/handlers/compile.interfaces.ts +++ b/lib/handlers/compile.interfaces.ts @@ -22,7 +22,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -import {BypassCache, ExecutionParams} from '../../types/compilation/compilation.interfaces.js'; +import {BypassCache, UnparsedExecutionParams} from '../../types/compilation/compilation.interfaces.js'; // IF YOU MODIFY ANYTHING HERE PLEASE UPDATE THE DOCUMENTATION! @@ -40,7 +40,7 @@ export type CompileRequestQueryArgs = { export type CompilationRequestArgs = { userArguments: string; compilerOptions: Record; - executeParameters: ExecutionParams; + executeParameters: UnparsedExecutionParams; filters: Record; tools: any; libraries: any[]; diff --git a/lib/handlers/compile.ts b/lib/handlers/compile.ts index aff431b02..086a773c6 100644 --- a/lib/handlers/compile.ts +++ b/lib/handlers/compile.ts @@ -41,6 +41,7 @@ import { BypassCache, ExecutionParams, FiledataPair, + UnparsedExecutionParams, } from '../../types/compilation/compilation.interfaces.js'; import {CompilerOverrideOptions} from '../../types/compilation/compiler-overrides.interfaces.js'; import {CompilerInfo, PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; @@ -381,7 +382,7 @@ export class CompileHandler implements ICompileHandler { filters: ParseFiltersAndOutputOptions, bypassCache = BypassCache.None, tools; - const execReqParams: ExecutionParams = {}; + const execReqParams: UnparsedExecutionParams = {}; let libraries: any[] = []; // IF YOU MODIFY ANYTHING HERE PLEASE UPDATE THE DOCUMENTATION! if (req.is('json')) { diff --git a/lib/handlers/health-check.ts b/lib/handlers/health-check.ts index cad3f0cba..f64fcdf6f 100644 --- a/lib/handlers/health-check.ts +++ b/lib/handlers/health-check.ts @@ -38,6 +38,7 @@ export class HealthCheckHandler { private readonly compilationQueue: CompilationQueue, private readonly filePath: any, private readonly compileHandler: ICompileHandler, + private readonly isExecutionWorker: boolean, ) { this.handle = this._handle.bind(this); } @@ -55,7 +56,7 @@ export class HealthCheckHandler { */ await this.compilationQueue.enqueue(async () => {}, {highPriority: true}); - if (!this.compileHandler.hasLanguages()) { + if (!this.isExecutionWorker && !this.compileHandler.hasLanguages()) { logger.error(`*** HEALTH CHECK FAILURE: no languages/compilers detected`); res.status(500).end(); return; diff --git a/lib/instructionsets.ts b/lib/instructionsets.ts index a1590b5ba..19e576f51 100644 --- a/lib/instructionsets.ts +++ b/lib/instructionsets.ts @@ -151,6 +151,10 @@ export class InstructionSets { target: [], path: [], }, + x86: { + target: [], + path: [], + }, amd64: { target: ['x86_64'], path: ['/x86_64'], diff --git a/lib/properties.ts b/lib/properties.ts index 3f9887eb6..ae6efda7b 100644 --- a/lib/properties.ts +++ b/lib/properties.ts @@ -48,6 +48,8 @@ function debug(str: string) { if (propDebug) logger.info(`prop: ${str}`); } +export type PropFunc = (s: string, a?: any) => any; + export function get(base: string, property: string, defaultValue: any): PropertyValue; export function get( base: string, @@ -279,8 +281,8 @@ export function setDebug(debug: boolean) { propDebug = debug; } -export function fakeProps(fake: Record): PropertyGetter { - return (prop: string, def: any) => (fake[prop] === undefined ? def : fake[prop]); +export function fakeProps(fake: Record): PropFunc { + return (prop, def) => (fake[prop] === undefined ? def : fake[prop]); } export function getRawProperties() { diff --git a/lib/toolchain-utils.ts b/lib/toolchain-utils.ts index 5e5cd99cb..b0a621ba5 100644 --- a/lib/toolchain-utils.ts +++ b/lib/toolchain-utils.ts @@ -42,7 +42,7 @@ export function getToolchainPathWithOptionsArr(compilerExe: string | null, optio const gxxname = options.find(elem => elem.includes(icc_style_toolchain_flag)); if (gxxname) { return path.resolve(path.dirname(gxxname.substring(11)), '..'); - } else if (typeof compilerExe === 'string' && compilerExe.includes('/g++')) { + } else if (typeof compilerExe === 'string' && (compilerExe.includes('/g++') || compilerExe.endsWith('-g++'))) { return path.resolve(path.dirname(compilerExe), '..'); } else { return false; diff --git a/package-lock.json b/package-lock.json index 0df1e4c47..15d79784b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@aws-sdk/client-dynamodb": "^3.675.0", "@aws-sdk/client-ec2": "^3.675.0", "@aws-sdk/client-s3": "^3.675.0", + "@aws-sdk/client-sqs": "^3.614.0", "@aws-sdk/client-ssm": "^3.675.0", "@aws-sdk/credential-providers": "^3.675.0", "@flatten-js/interval-tree": "^1.1.3", @@ -75,6 +76,7 @@ "winston-loki": "^6.1.3", "winston-papertrail": "^1.0.5", "winston-transport": "^4.8.0", + "ws": "^8.18.0", "yaml": "^2.6.0" }, "devDependencies": { @@ -576,6 +578,674 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-sqs": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.614.0.tgz", + "integrity": "sha512-jCeHIlfwBSuoKN7Nf+DHYbH1eBuWALf+o9WaK8J+xhU0VE6G0DGdjHhjW1aTGTchBntARzqzMqwvY1e18Ac1zQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.614.0", + "@aws-sdk/client-sts": "3.614.0", + "@aws-sdk/core": "3.614.0", + "@aws-sdk/credential-provider-node": "3.614.0", + "@aws-sdk/middleware-host-header": "3.609.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.609.0", + "@aws-sdk/middleware-sdk-sqs": "3.614.0", + "@aws-sdk/middleware-user-agent": "3.614.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.6", + "@smithy/fetch-http-handler": "^3.2.1", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/md5-js": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.3", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.9", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.2", + "@smithy/protocol-http": "^4.0.3", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.9", + "@smithy/util-defaults-mode-node": "^3.0.9", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/client-sso": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.614.0.tgz", + "integrity": "sha512-p5pyYaxRzBttjBkqfc8i3K7DzBdTg3ECdVgBo6INIUxfvDy0J8QUE8vNtCgvFIkq+uPw/8M+Eo4zzln7anuO0Q==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.614.0", + "@aws-sdk/middleware-host-header": "3.609.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.609.0", + "@aws-sdk/middleware-user-agent": "3.614.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.6", + "@smithy/fetch-http-handler": "^3.2.1", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.3", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.9", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.2", + "@smithy/protocol-http": "^4.0.3", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.9", + "@smithy/util-defaults-mode-node": "^3.0.9", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.614.0.tgz", + "integrity": "sha512-BI1NWcpppbHg/28zbUg54dZeckork8BItZIcjls12vxasy+p3iEzrJVG60jcbUTTsk3Qc1tyxNfrdcVqx0y7Ww==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.614.0", + "@aws-sdk/credential-provider-node": "3.614.0", + "@aws-sdk/middleware-host-header": "3.609.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.609.0", + "@aws-sdk/middleware-user-agent": "3.614.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.6", + "@smithy/fetch-http-handler": "^3.2.1", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.3", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.9", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.2", + "@smithy/protocol-http": "^4.0.3", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.9", + "@smithy/util-defaults-mode-node": "^3.0.9", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.614.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/client-sts": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.614.0.tgz", + "integrity": "sha512-i6QmaVA1KHHYNnI2VYQy/sc31rLm4+jSp8b/YbQpFnD0w3aXsrEEHHlxek45uSkHb4Nrj1omFBVy/xp1WVYx2Q==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.614.0", + "@aws-sdk/core": "3.614.0", + "@aws-sdk/credential-provider-node": "3.614.0", + "@aws-sdk/middleware-host-header": "3.609.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.609.0", + "@aws-sdk/middleware-user-agent": "3.614.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.2.6", + "@smithy/fetch-http-handler": "^3.2.1", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.3", + "@smithy/middleware-endpoint": "^3.0.5", + "@smithy/middleware-retry": "^3.0.9", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.2", + "@smithy/protocol-http": "^4.0.3", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.9", + "@smithy/util-defaults-mode-node": "^3.0.9", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/core": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.614.0.tgz", + "integrity": "sha512-BUuS5/1YkgmKc4J0bg83XEtMyDHVyqG2QDzfmhYe8gbOIZabUl1FlrFVwhCAthtrrI6MPGTQcERB4BtJKUSplw==", + "dependencies": { + "@smithy/core": "^2.2.6", + "@smithy/protocol-http": "^4.0.3", + "@smithy/signature-v4": "^3.1.2", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "fast-xml-parser": "4.2.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.609.0.tgz", + "integrity": "sha512-v69ZCWcec2iuV9vLVJMa6fAb5xwkzN4jYIT8yjo2c4Ia/j976Q+TPf35Pnz5My48Xr94EFcaBazrWedF+kwfuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.614.0.tgz", + "integrity": "sha512-YIEjlNUKb3Vo/iTnGAPdsiDC3FUUnNoex2OwU8LmR7AkYZiWdB8nx99DfgkkY+OFMUpw7nKD2PCOtuFONelfGA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.1", + "@smithy/node-http-handler": "^3.1.2", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.0.3", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.0.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.614.0.tgz", + "integrity": "sha512-KfLuLFGwlvFSZ2MuzYwWGPb1y5TeiwX5okIDe0aQ1h10oD3924FXbN+mabOnUHQ8EFcGAtCaWbrC86mI7ktC6A==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.609.0", + "@aws-sdk/credential-provider-http": "3.614.0", + "@aws-sdk/credential-provider-process": "3.614.0", + "@aws-sdk/credential-provider-sso": "3.614.0", + "@aws-sdk/credential-provider-web-identity": "3.609.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.614.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.614.0.tgz", + "integrity": "sha512-4J6gPEuFZP0mkWq5E//oMS1vrmMM88iNNcv7TEljYnsc6JTAlKejCyFwx6CN+nkIhmIZsl06SXIhBemzBdBPfg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.609.0", + "@aws-sdk/credential-provider-http": "3.614.0", + "@aws-sdk/credential-provider-ini": "3.614.0", + "@aws-sdk/credential-provider-process": "3.614.0", + "@aws-sdk/credential-provider-sso": "3.614.0", + "@aws-sdk/credential-provider-web-identity": "3.609.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.614.0.tgz", + "integrity": "sha512-Q0SI0sTRwi8iNODLs5+bbv8vgz8Qy2QdxbCHnPk/6Cx6LMf7i3dqmWquFbspqFRd8QiqxStrblwxrUYZi09tkA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.614.0.tgz", + "integrity": "sha512-55+gp0JY4451cWI1qXmVMFM0GQaBKiQpXv2P0xmd9P3qLDyeFUSEW8XPh0d2lb1ICr6x4s47ynXVdGCIv2mXMg==", + "dependencies": { + "@aws-sdk/client-sso": "3.614.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.609.0.tgz", + "integrity": "sha512-U+PG8NhlYYF45zbr1km3ROtBMYqyyj/oK8NRp++UHHeuavgrP+4wJ4wQnlEaKvJBjevfo3+dlIBcaeQ7NYejWg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.609.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.609.0.tgz", + "integrity": "sha512-iTKfo158lc4jLDfYeZmYMIBHsn8m6zX+XB6birCSNZ/rrlzAkPbGE43CNdKfvjyWdqgLMRXF+B+OcZRvqhMXPQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.609.0.tgz", + "integrity": "sha512-6sewsYB7/o/nbUfA99Aa/LokM+a/u4Wpm/X2o0RxOsDtSB795ObebLJe2BxY5UssbGaWkn7LswyfvrdZNXNj1w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.614.0.tgz", + "integrity": "sha512-xUxh0UPQiMTG6E31Yvu6zVYlikrIcFDKljM11CaatInzvZubGTGiX0DjpqRlfGzUNsuPc/zNrKwRP2+wypgqIw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@smithy/protocol-http": "^4.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.614.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-endpoints": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.614.0.tgz", + "integrity": "sha512-wK2cdrXHH4oz4IomV/yrGkftU9A+ITB6nFL+rxxyO78is2ifHJpFdV4aqk4LSkXYPi6CXWNru/Dqc7yiKXgJPw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@smithy/signature-v4": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.1.2.tgz", + "integrity": "sha512-3BcPylEsYtD0esM4Hoyml/+s7WP2LFhcM3J2AGdcL2vx9O60TtfpDOL72gjb4lU8NeRPeKAwR77YNyyGvMbuEA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@smithy/signature-v4/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@smithy/util-stream": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.6.tgz", + "integrity": "sha512-w9i//7egejAIvplX821rPWWgaiY1dxsQUw0hXX7qwa/uZ9U3zplqTQ871jWadkcVB9gFDhkPWYVZf4yfFbZ0xA==", + "dependencies": { + "@smithy/fetch-http-handler": "^3.2.1", + "@smithy/node-http-handler": "^3.1.2", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/fast-xml-parser": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", + "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", + "funding": [ + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + }, + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-ssm": { "version": "3.675.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-ssm/-/client-ssm-3.675.0.tgz", @@ -1153,6 +1823,34 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-sqs": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.614.0.tgz", + "integrity": "sha512-TFBbXEMnzBqGVPatL5Pg8a3kPOUrhSWxXXWZjr6S4D5ffCJnCNSzfi09SUoehe6uYmTQlS7AAqugTIFdRnA/ww==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/smithy-client": "^3.1.7", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-sqs/node_modules/@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@aws-sdk/middleware-ssec": { "version": "3.667.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.667.0.tgz", @@ -15973,6 +16671,26 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 2c3276b35..918f651e8 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@aws-sdk/client-dynamodb": "^3.675.0", "@aws-sdk/client-ec2": "^3.675.0", "@aws-sdk/client-s3": "^3.675.0", + "@aws-sdk/client-sqs": "^3.614.0", "@aws-sdk/client-ssm": "^3.675.0", "@aws-sdk/credential-providers": "^3.675.0", "@flatten-js/interval-tree": "^1.1.3", @@ -84,6 +85,7 @@ "winston-loki": "^6.1.3", "winston-papertrail": "^1.0.5", "winston-transport": "^4.8.0", + "ws": "^8.18.0", "yaml": "^2.6.0" }, "devDependencies": { diff --git a/static/panes/compiler.ts b/static/panes/compiler.ts index 750153606..ca9ad0e38 100644 --- a/static/panes/compiler.ts +++ b/static/panes/compiler.ts @@ -1181,7 +1181,7 @@ export class Compiler extends MonacoPane { } compilerIsVisible(compiler: CompilerInfo): boolean { - return !!compiler.supportsExecute; + return !!(compiler.supportsExecute || compiler.supportsBinary); } getEditorIdByFilename(filename: string): number | null { @@ -315,7 +315,7 @@ export class Executor extends Pane { } compileFromEditorSource(options: CompilationRequestOptions, bypassCache?: BypassCache): void { - if (!this.compiler?.supportsExecute) { + if (!this.compiler || !this.compilerIsVisible(this.compiler)) { this.alertSystem.notify('This compiler (' + this.compiler?.name + ') does not support execution', { group: 'execution', }); @@ -629,6 +629,9 @@ export class Executor extends Pane { if (!result.didExecute) { this.executionStatusSection.append($('
').text('Could not execute the program')); + if (execStderr.length > 0) { + this.handleOutput(execStderr, this.executionStatusSection, this.normalAnsiToHtml, false); + } this.executionStatusSection.append($('
').text('Compiler returned: ' + buildResultCode)); } // reset stream styles @@ -1281,9 +1284,9 @@ export class Executor extends Pane { ); if (!allCompilers) return []; - const hasAtLeastOneExecuteSupported = Object.values(allCompilers).some(compiler => { - return compiler.supportsExecute !== false; - }); + const hasAtLeastOneExecuteSupported = Object.values(allCompilers).some(compiler => + this.compilerIsVisible(compiler), + ); if (!hasAtLeastOneExecuteSupported) { this.compiler = null; @@ -1292,7 +1295,7 @@ export class Executor extends Pane { return Object.values(allCompilers).filter(compiler => { return ( - (compiler.hidden !== true && compiler.supportsExecute !== false) || + (compiler.hidden !== true && this.compilerIsVisible(compiler)) || (this.compiler && compiler.id === this.compiler.id) ); }); diff --git a/test/base-compiler-tests.ts b/test/base-compiler-tests.ts index 7bb9cb551..be1010017 100644 --- a/test/base-compiler-tests.ts +++ b/test/base-compiler-tests.ts @@ -22,13 +22,15 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -import {beforeAll, describe, expect, it} from 'vitest'; +import {afterAll, beforeAll, describe, expect, it} from 'vitest'; import {BaseCompiler} from '../lib/base-compiler.js'; import {BuildEnvSetupBase} from '../lib/buildenvsetup/index.js'; import {CompilationEnvironment} from '../lib/compilation-env.js'; import {ClangCompiler} from '../lib/compilers/clang.js'; +import {RustCompiler} from '../lib/compilers/rust.js'; import {Win32Compiler} from '../lib/compilers/win32.js'; +import * as props from '../lib/properties.js'; import {splitArguments} from '../lib/utils.js'; import {CompilerOverrideType, ConfiguredOverrides} from '../types/compilation/compiler-overrides.interfaces.js'; import {CompilerInfo} from '../types/compiler.interfaces.js'; @@ -44,6 +46,7 @@ import { const languages = { 'c++': {id: 'c++'}, + rust: {id: 'rust'}, } as const; describe('Basic compiler invariants', () => { @@ -752,3 +755,66 @@ describe('Target hints', () => { expect(iset).toBe('riscv64'); }); }); + +describe('Rust overrides', () => { + let ce: CompilationEnvironment; + const executingCompilerInfo = makeFakeCompilerInfo({ + remote: { + target: '', + path: '', + cmakePath: '', + }, + semver: 'nightly', + lang: 'rust', + ldPath: [], + libPath: [], + supportsExecute: true, + supportsBinary: true, + options: '', + }); + + beforeAll(() => { + ce = makeCompilationEnvironment({ + languages, + }); + props.initialize(path.resolve('./test/test-properties/rust'), ['local']); + }); + + afterAll(() => { + props.reset(); + }); + + it('Empty options check', () => { + const compiler = new RustCompiler(executingCompilerInfo, ce); + expect(compiler.changeOptionsBasedOnOverrides([], [])).toEqual([]); + }); + + it('Should change linker if target is aarch64', () => { + const compiler = new RustCompiler(executingCompilerInfo, ce); + const originalOptions = compiler.optionsForFilter( + { + binary: true, + execute: true, + }, + 'output.txt', + [], + ); + expect(originalOptions).toEqual([ + '-C', + 'debuginfo=1', + '-o', + 'output.txt', + '--crate-type', + 'bin', + '-Clinker=/usr/amd64/bin/gcc', + ]); + expect( + compiler.changeOptionsBasedOnOverrides(originalOptions, [ + { + name: CompilerOverrideType.arch, + value: 'aarch64-linux-something', + }, + ]), + ).toEqual(['-C', 'debuginfo=1', '-o', 'output.txt', '--crate-type', 'bin', '-Clinker=/usr/aarch64/bin/gcc']); + }); +}); diff --git a/test/compilation-env.ts b/test/compilation-env.ts index 68a72e570..786f8ab24 100644 --- a/test/compilation-env.ts +++ b/test/compilation-env.ts @@ -43,28 +43,28 @@ describe('Compilation environment', () => { it('Should cache by default', async () => { // TODO: Work will need to be done here when CompilationEnvironment's constructor is typed better - const ce = new CompilationEnvironment(compilerProps, undefined, undefined); + const ce = new CompilationEnvironment(compilerProps, fakeProps({}), undefined, undefined); await expect(ce.cacheGet('foo')).resolves.toBeNull(); await ce.cachePut('foo', {res: 'bar'}, undefined); await expect(ce.cacheGet('foo')).resolves.toEqual({res: 'bar'}); await expect(ce.cacheGet('baz')).resolves.toBeNull(); }); it('Should cache when asked', async () => { - const ce = new CompilationEnvironment(compilerProps, undefined, true); + const ce = new CompilationEnvironment(compilerProps, fakeProps({}), undefined, true); await expect(ce.cacheGet('foo')).resolves.toBeNull(); await ce.cachePut('foo', {res: 'bar'}, undefined); await expect(ce.cacheGet('foo')).resolves.toEqual({res: 'bar'}); }); it("Shouldn't cache when asked", async () => { // TODO: Work will need to be done here when CompilationEnvironment's constructor is typed better - const ce = new CompilationEnvironment(compilerProps, undefined, false); + const ce = new CompilationEnvironment(compilerProps, fakeProps({}), undefined, false); await expect(ce.cacheGet('foo')).resolves.toBeNull(); await ce.cachePut('foo', {res: 'bar'}, undefined); await expect(ce.cacheGet('foo')).resolves.toBeNull(); }); it('Should filter bad options', () => { // TODO: Work will need to be done here when CompilationEnvironment's constructor is typed better - const ce = new CompilationEnvironment(compilerProps, undefined, undefined); + const ce = new CompilationEnvironment(compilerProps, fakeProps({}), undefined, undefined); expect(ce.findBadOptions(['-O3', '-flto'])).toEqual([]); expect(ce.findBadOptions(['-O3', '-plugin'])).toEqual(['-plugin']); }); diff --git a/test/execution-triple-tests.ts b/test/execution-triple-tests.ts new file mode 100644 index 000000000..8f7303eca --- /dev/null +++ b/test/execution-triple-tests.ts @@ -0,0 +1,78 @@ +import {describe, expect, it} from 'vitest'; + +import {BinaryInfoLinux} from '../lib/binaries/binary-utils.js'; +import {BaseExecutionTriple, ExecutionSpecialty} from '../lib/execution/base-execution-triple.js'; +import {getExecutionTriplesForCurrentHost, matchesCurrentHost} from '../lib/execution/execution-triple.js'; + +describe('Execution triple utils', () => { + it('default always matches', () => { + const triples: BaseExecutionTriple[] = getExecutionTriplesForCurrentHost(); + expect(matchesCurrentHost(triples[0])).toEqual(true); + }); + it('can parse a thing', () => { + const triple = new BaseExecutionTriple(); + triple.parse('aarch64-linux-cpu'); + expect(triple.instructionSet).toEqual('aarch64'); + expect(triple.os).toEqual('linux'); + expect(triple.specialty).toEqual(ExecutionSpecialty.cpu); + }); + it('would parse nvgpu', () => { + const triple = new BaseExecutionTriple(); + triple.parse('amd64-linux-nvgpu'); + expect(triple.instructionSet).toEqual('amd64'); + expect(triple.os).toEqual('linux'); + expect(triple.specialty).toEqual(ExecutionSpecialty.nvgpu); + }); + it('recognizes aarch64', () => { + const info = BinaryInfoLinux.parseFileInfo( + 'ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 5.17.0, with debug_info, not stripped', + ); + expect(info?.instructionSet).toEqual('aarch64'); + expect(info?.os).toEqual('linux'); + }); + it('recognizes arm32', () => { + const info = BinaryInfoLinux.parseFileInfo( + 'ELF 32-bit LSB executable, ARM, EABI5 version 1 (GNU/Linux), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 4.4.255, with debug_info, not stripped', + ); + expect(info?.instructionSet).toEqual('arm32'); + expect(info?.os).toEqual('linux'); + }); + it('recognizes lin64', () => { + const info = BinaryInfoLinux.parseFileInfo( + 'ELF 64-bit LSB executable, x86-64, version 1 (GNU/Linux), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=9def7a0f68cf33177d0d8566b152af24d9ec1ac4, for GNU/Linux 3.2.0, stripped', + ); + expect(info?.instructionSet).toEqual('amd64'); + expect(info?.os).toEqual('linux'); + }); + it('recognizes win64', () => { + const info = BinaryInfoLinux.parseFileInfo( + 'PE32+ executable (DLL) (console) x86-64 (stripped to external PDB), for MS Windows, 12 sections', + ); + expect(info?.instructionSet).toEqual('amd64'); + expect(info?.os).toEqual('win32'); + }); + it('recognizes win alternatives', () => { + const filteredLine = BinaryInfoLinux.removeComments( + 'PE32+ executable (DLL) (console) Aarch64 (stripped to external PDB)', + ); + expect(filteredLine).toEqual('PE32+ executable Aarch64'); + + const info = BinaryInfoLinux.parseFileInfo( + 'PE32+ executable (DLL) (console) Aarch64 (stripped to external PDB), for MS Windows, 12 sections', + ); + expect(info?.instructionSet).toEqual('aarch64'); + expect(info?.os).toEqual('win32'); + }); + it('recognizes win32', () => { + const info = BinaryInfoLinux.parseFileInfo('PE32 executable (GUI) Intel 80386, for MS Windows, 4 sections'); + expect(info?.instructionSet).toEqual('x86'); + expect(info?.os).toEqual('win32'); + }); + it('recognizes avr', () => { + const info = BinaryInfoLinux.parseFileInfo( + 'ELF 32-bit LSB executable, Atmel AVR 8-bit, version 1 (SYSV), statically linked, with debug_info, not stripped', + ); + expect(info?.instructionSet).toEqual('avr'); + expect(info?.os).toEqual('linux'); + }); +}); diff --git a/test/handlers/health-check-tests.ts b/test/handlers/health-check-tests.ts index da7889d51..426de43bd 100644 --- a/test/handlers/health-check-tests.ts +++ b/test/handlers/health-check-tests.ts @@ -40,7 +40,7 @@ describe('Health checks', () => { }; compilationQueue = new CompilationQueue(1, 0, 0); app = express(); - app.use('/hc', new HealthCheckHandler(compilationQueue, '', compileHandlerMock).handle); + app.use('/hc', new HealthCheckHandler(compilationQueue, '', compileHandlerMock, false).handle); }); it('should respond with OK', async () => { @@ -67,7 +67,7 @@ describe('Health checks without lang/comp', () => { }; compilationQueue = new CompilationQueue(1, 0, 0); app = express(); - app.use('/hc', new HealthCheckHandler(compilationQueue, '', compileHandlerMock).handle); + app.use('/hc', new HealthCheckHandler(compilationQueue, '', compileHandlerMock, false).handle); }); it('should respond with error', async () => { @@ -75,6 +75,24 @@ describe('Health checks without lang/comp', () => { }); }); +describe('Health checks without lang/comp but in execution worker mode', () => { + let app; + let compilationQueue; + + beforeEach(() => { + const compileHandlerMock = { + hasLanguages: () => false, + }; + compilationQueue = new CompilationQueue(1, 0, 0); + app = express(); + app.use('/hc', new HealthCheckHandler(compilationQueue, '', compileHandlerMock, true).handle); + }); + + it('should respond with ok', async () => { + await request(app).get('/hc').expect(200); + }); +}); + describe('Health checks on disk', () => { let app; @@ -85,8 +103,8 @@ describe('Health checks on disk', () => { const compilationQueue = new CompilationQueue(1, 0, 0); app = express(); - app.use('/hc', new HealthCheckHandler(compilationQueue, '/fake/.nonexist', compileHandlerMock).handle); - app.use('/hc2', new HealthCheckHandler(compilationQueue, '/fake/.health', compileHandlerMock).handle); + app.use('/hc', new HealthCheckHandler(compilationQueue, '/fake/.nonexist', compileHandlerMock, false).handle); + app.use('/hc2', new HealthCheckHandler(compilationQueue, '/fake/.health', compileHandlerMock, false).handle); mockfs({ '/fake': { diff --git a/test/library-tests.ts b/test/library-tests.ts index 4c98df2d4..401ce4bd8 100644 --- a/test/library-tests.ts +++ b/test/library-tests.ts @@ -164,7 +164,7 @@ describe('Library directories (c++)', () => { expect(fmtpaths).not.toContain('-I/tmp/compiler-explorer-compiler-123/fmt/include'); expect(fmtpaths).toContain('-I/opt/compiler-explorer/libs/fmt/1.0/include'); - const qtpaths = (compiler as any).getIncludeArguments( + const qtpaths = (compiler as BaseCompiler).getIncludeArguments( [{id: 'qt', version: '660'}], '/tmp/compiler-explorer-compiler-123', ); @@ -177,8 +177,16 @@ describe('Library directories (c++)', () => { it('should set LD_LIBRARY_PATH when executing', () => { (compiler as any).sandboxType = 'nsjail'; - const qtpaths = (compiler as any).getSharedLibraryPathsAsLdLibraryPathsForExecution( - [{id: 'qt', version: '660'}], + const qtpaths = (compiler as BaseCompiler).getSharedLibraryPathsAsLdLibraryPathsForExecution( + { + libraries: [{id: 'qt', version: '660'}], + compiler: undefined, + source: '', + options: [], + backendOptions: undefined, + tools: [], + files: [], + }, '/tmp/compiler-explorer-compiler-123', ); diff --git a/test/test-properties/rust/rust.local.properties b/test/test-properties/rust/rust.local.properties new file mode 100644 index 000000000..cd743da05 --- /dev/null +++ b/test/test-properties/rust/rust.local.properties @@ -0,0 +1,2 @@ +linker=/usr/amd64/bin/gcc +aarch64linker=/usr/aarch64/bin/gcc diff --git a/test/tool-tests.ts b/test/tool-tests.ts index 9e22388dd..16917949e 100644 --- a/test/tool-tests.ts +++ b/test/tool-tests.ts @@ -80,7 +80,7 @@ describe('CompilerDropInTool', () => { ]); }); - it('Should not support riscv gcc compilers', () => { + it('Should maybe support riscv gcc compilers', () => { const tool = new CompilerDropinTool({} as ToolInfo, {} as ToolEnv); const compilationInfo = { @@ -95,7 +95,11 @@ describe('CompilerDropInTool', () => { const sourcefile = 'example.cpp'; const orderedArgs = tool.getOrderedArguments(compilationInfo, includeflags, [], args, sourcefile); - expect(orderedArgs).toEqual(false); + // note: toolchain twice because reasons, see CompilerDropinTool getOrderedArguments() + expect(orderedArgs).toEqual([ + '--gcc-toolchain=' + path.resolve('/opt/compiler-explorer/riscv64/gcc-8.2.0/riscv64-unknown-linux-gnu'), + '--gcc-toolchain=' + path.resolve('/opt/compiler-explorer/riscv64/gcc-8.2.0/riscv64-unknown-linux-gnu'), + ]); }); it('Should support ICC compilers', () => { diff --git a/test/utils.ts b/test/utils.ts index 174f46177..a0690024d 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -41,7 +41,7 @@ import {Language} from '../types/languages.interfaces.js'; export function makeCompilationEnvironment(options: Record): CompilationEnvironment { const compilerProps = new CompilerProps(options.languages, fakeProps(options.props || {})); const compilationQueue = options.queue || new CompilationQueue(options.concurrency || 1, options.timeout, 100_000); - return new CompilationEnvironment(compilerProps, compilationQueue, options.doCache); + return new CompilationEnvironment(compilerProps, fakeProps({}), compilationQueue, options.doCache); } export function makeFakeCompilerInfo(props: Partial): CompilerInfo { diff --git a/types/compilation/compilation.interfaces.ts b/types/compilation/compilation.interfaces.ts index ce5693478..28b61e286 100644 --- a/types/compilation/compilation.interfaces.ts +++ b/types/compilation/compilation.interfaces.ts @@ -48,6 +48,12 @@ export type ActiveTool = { stdin: string; }; +export type UnparsedExecutionParams = { + args?: string | string[]; + stdin?: string; + runtimeTools?: ConfiguredRuntimeTools; +}; + export type ExecutionParams = { args?: string[]; stdin?: string; @@ -109,7 +115,7 @@ export type CompilationRequestOptions = { customOutputFilename?: string; overrides?: ConfiguredOverrides; }; - executeParameters: ExecutionParams; + executeParameters: UnparsedExecutionParams; filters: ParseFiltersAndOutputOptions; tools: ActiveTool[]; libraries: SelectedLibraryVersion[]; diff --git a/types/instructionsets.ts b/types/instructionsets.ts index 96d1d1790..733e82671 100644 --- a/types/instructionsets.ts +++ b/types/instructionsets.ts @@ -26,6 +26,7 @@ export const InstructionSetsList = [ '6502', 'aarch64', 'amd64', + 'x86', 'arm32', 'avr', 'beam',