mirrors/compiler-explorer
1
0
Fork 0
mirror of https://github.com/compiler-explorer/compiler-explorer.git synced 2025-12-27 07:04:04 -05:00
Code Issues Packages Projects Releases Wiki Activity

cmake on windows improvements (#8174)

Browse Source
This commit is contained in:
Patrick Quist
2025-10-11 15:55:26 +02:00
committed by GitHub
parent fb57aa7b54
commit f3571b9bc4
6 changed files with 215 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
etc/cewrapper/compilers-and-tools-win.json Normal file
View File

@@ -0,0 +1,7 @@
{
"use_appcontainer": false,
"pids_max": 72,
"mem_max": 1342177280,
"allowed_paths": [],
"allowed_registry": []
}

2
etc/config/compiler-explorer.amazonwin.properties
View File

@@ -44,6 +44,8 @@ formatter.clangformat.type=clangformat
compilationStatsNotifier=S3(compiler-explorer-logs,compile-stats,us-east-1,15m)
cewrapper.config.execute=etc/cewrapper/compilers-and-tools-win.json
execqueue.queue_url=https://sqs.us-east-1.amazonaws.com/052730242331/prod-execqueue
execqueue.events_url=wss://events.compiler-explorer.com/prod
execqueue.is_worker=false

2
etc/config/execution.amazonwin.properties
View File

@@ -2,3 +2,5 @@ sandboxType=cewrapper
executionType=cewrapper
cewrapper=C:/cewrapper/cewrapper.exe
# unbufferStdoutExe=
cewrapper.config.execute=etc/cewrapper/compilers-and-tools-win.json

299
lib/base-compiler.ts
View File

@@ -494,7 +494,9 @@ export class BaseCompiler {
});
}
if (options.createAndUseTempDir) fs.rm(options.customCwd!, {recursive: true, force: true}).catch(() => {});
if (options.createAndUseTempDir) {
fs.rm(options.customCwd!, {recursive: true, force: true}).catch(() => {});
}
return result;
}
@@ -2547,9 +2549,9 @@ export class BaseCompiler {
return this.checkOutputFileAndDoPostProcess(asmResult, outputFilename, filters, backendOptions.produceOptInfo);
}
doTempfolderCleanup(buildResult: BuildResult | CompilationResult) {
async doTempfolderCleanup(buildResult: BuildResult | CompilationResult) {
if (buildResult.dirPath && !this.delayCleanupTemp) {
fs.rm(buildResult.dirPath, {recursive: true, force: true}).catch(() => {});
await fs.rm(buildResult.dirPath, {recursive: true, force: true}).catch(() => {});
}
buildResult.dirPath = undefined;
}
@@ -2730,8 +2732,8 @@ export class BaseCompiler {
const outputFilename = this.getExecutableFilename(path.join(dirPath, 'build'), this.outputFilebase, cacheKey);
let fullResult: CompilationResult = bypassExecutionCache(bypassCache)
? null
let fullResult: CompilationResult = bypassCompilationCache(bypassCache)
? undefined
: await this.loadPackageWithExecutable(cacheKey, executablePackageHash, dirPath);
if (fullResult) {
fullResult.retreivedFromCache = true;
@@ -2741,151 +2743,177 @@ export class BaseCompiler {
delete fullResult.dirPath;
fullResult.executableFilename = outputFilename;
} else {
let writeSummary;
try {
writeSummary = await this.writeAllFilesCMake(dirPath, cacheKey.source, files, cacheKey.filters);
} catch (e) {
return this.handleUserError(e, dirPath);
}
const queueTime = performance.now();
const moreResult = await this.env.enqueue(async () => {
const start = performance.now();
compilationQueueTimeHistogram.observe((start - queueTime) / 1000);
const execParams = this.getDefaultExecOptions();
execParams.appHome = dirPath;
execParams.customCwd = path.join(dirPath, 'build');
let writeSummary;
try {
writeSummary = await this.writeAllFilesCMake(dirPath, cacheKey.source, files, cacheKey.filters);
} catch (e) {
return this.handleUserError(e, dirPath);
}
await fs.mkdir(execParams.customCwd);
const execParams = this.getDefaultExecOptions();
execParams.appHome = dirPath;
execParams.customCwd = path.join(dirPath, 'build');
const makeExecParams = this.createCmakeExecParams(execParams, dirPath, libsAndOptions, toolchainPath);
await fs.mkdir(execParams.customCwd);
fullResult = {
code: 0,
timedOut: false,
stdout: [],
stderr: [],
buildsteps: [],
inputFilename: writeSummary.inputFilename,
executableFilename: outputFilename,
};
const makeExecParams = this.createCmakeExecParams(execParams, dirPath, libsAndOptions, toolchainPath);
fullResult.downloads = await this.setupBuildEnvironment(cacheKey, dirPath, true);
const toolchainparam = this.getCMakeExtToolchainParam(parsedRequest.backendOptions.overrides || []);
const cmakeArgs = splitArguments(parsedRequest.backendOptions.cmakeArgs);
const partArgs: string[] = [
toolchainparam,
...this.getExtraCMakeArgs(parsedRequest),
...cmakeArgs,
'..',
].filter(Boolean); // filter out empty args
const useNinja = this.env.ceProps('useninja');
const fullArgs: string[] = useNinja ? ['-GNinja'].concat(partArgs) : partArgs;
const cmd = this.env.ceProps('cmake') as string;
assert(cmd, 'No cmake command found');
const cmakeStepResult = await this.doBuildstepAndAddToResult(
fullResult,
'cmake',
cmd,
fullArgs,
makeExecParams,
);
if (cmakeStepResult.code !== 0) {
fullResult.result = {
dirPath,
const result: CompilationResult = {
code: 0,
timedOut: false,
stdout: [],
stderr: [],
okToCache: false,
code: cmakeStepResult.code,
asm: [{text: '<Build failed>'}],
buildsteps: [],
inputFilename: writeSummary.inputFilename,
executableFilename: outputFilename,
};
fullResult.result.compilationOptions = this.getUsedEnvironmentVariableFlags(makeExecParams);
return fullResult;
}
const makeStepResult = await this.doBuildstepAndAddToResult(
fullResult,
'build',
cmd,
['--build', '.'],
execParams,
);
result.downloads = await this.setupBuildEnvironment(cacheKey, dirPath, true);
if (makeStepResult.code !== 0) {
fullResult.result = {
dirPath,
timedOut: false,
stdout: [],
stderr: [],
okToCache: false,
code: makeStepResult.code,
asm: [{text: '<Build failed>'}],
};
return fullResult;
}
const toolchainparam = this.getCMakeExtToolchainParam(parsedRequest.backendOptions.overrides || []);
fullResult.result = {
dirPath,
code: 0,
timedOut: false,
stdout: [],
stderr: [],
okToCache: true,
compilationOptions: this.getUsedEnvironmentVariableFlags(makeExecParams),
};
const cmakeArgs = splitArguments(parsedRequest.backendOptions.cmakeArgs);
const partArgs: string[] = [
toolchainparam,
...this.getExtraCMakeArgs(parsedRequest),
...cmakeArgs,
'..',
].filter(Boolean); // filter out empty args
const useNinja = this.env.ceProps('useninja');
const fullArgs: string[] = useNinja ? ['-GNinja'].concat(partArgs) : partArgs;
if (!parsedRequest.backendOptions.skipAsm) {
const [asmResult] = await this.checkOutputFileAndDoPostProcess(
fullResult.result,
outputFilename,
cacheKey.filters,
const cmd = this.env.ceProps('cmake') as string;
assert(cmd, 'No cmake command found');
const cmakeStepResult = await this.doBuildstepAndAddToResult(
result,
'cmake',
cmd,
fullArgs,
makeExecParams,
);
fullResult.result = asmResult;
}
fullResult.code = 0;
if (fullResult.buildsteps) {
_.each(fullResult.buildsteps, step => {
fullResult.code += step.code;
});
}
if (cmakeStepResult.code !== 0) {
result.result = {
dirPath,
timedOut: false,
stdout: [],
stderr: [],
okToCache: false,
code: cmakeStepResult.code,
asm: [{text: '<Build failed>'}],
};
result.result.compilationOptions = this.getUsedEnvironmentVariableFlags(makeExecParams);
compilationTimeHistogram.observe((performance.now() - start) / 1000);
return result;
}
await this.storePackageWithExecutable(executablePackageHash, dirPath, fullResult);
const makeStepResult = await this.doBuildstepAndAddToResult(
result,
'build',
cmd,
['--build', '.'],
execParams,
);
if (makeStepResult.code !== 0) {
result.result = {
dirPath,
timedOut: false,
stdout: [],
stderr: [],
okToCache: false,
code: makeStepResult.code,
asm: [{text: '<Build failed>'}],
};
compilationTimeHistogram.observe((performance.now() - start) / 1000);
return result;
}
result.result = {
dirPath,
code: 0,
timedOut: false,
stdout: [],
stderr: [],
okToCache: true,
compilationOptions: this.getUsedEnvironmentVariableFlags(makeExecParams),
};
if (!parsedRequest.backendOptions.skipAsm) {
const [asmResult] = await this.checkOutputFileAndDoPostProcess(
result.result,
outputFilename,
cacheKey.filters,
);
result.result = asmResult;
}
result.code = 0;
if (result.buildsteps) {
_.each(result.buildsteps, step => {
result.code += step.code;
});
}
await this.storePackageWithExecutable(executablePackageHash, dirPath, result);
compilationTimeHistogram.observe((performance.now() - start) / 1000);
return result;
});
if (moreResult) fullResult = moreResult;
}
if (fullResult.result) {
fullResult.result.dirPath = dirPath;
if (doExecute && fullResult.result.code === 0) {
const execTriple = await RemoteExecutionQuery.guessExecutionTripleForBuildresult({
...fullResult,
downloads: fullResult.downloads || [],
executableFilename: outputFilename,
compilationOptions: fullResult.compilationOptions || [],
});
if (matchesCurrentHost(execTriple)) {
fullResult.execResult = await this.runExecutable(outputFilename, executeOptions, dirPath);
fullResult.didExecute = true;
// Check if executable exists before trying to run it
if (!(await utils.fileExists(outputFilename))) {
fullResult.execResult = {
code: -1,
okToCache: false,
stdout: [],
stderr: [{text: `Executable not found: ${utils.maskRootdir(outputFilename)}`}],
execTime: 0,
timedOut: false,
};
fullResult.didExecute = false;
} else {
if (await RemoteExecutionQuery.isPossible(execTriple)) {
fullResult.execResult = await this.runExecutableRemotely(
executablePackageHash,
executeOptions,
execTriple,
);
const execTriple = await RemoteExecutionQuery.guessExecutionTripleForBuildresult({
...fullResult,
downloads: fullResult.downloads || [],
executableFilename: outputFilename,
compilationOptions: fullResult.compilationOptions || [],
});
if (matchesCurrentHost(execTriple)) {
fullResult.execResult = await this.runExecutable(outputFilename, executeOptions, dirPath);
fullResult.didExecute = true;
} else {
fullResult.execResult = {
code: -1,
okToCache: false,
stdout: [],
stderr: [{text: `No execution available for ${execTriple.toString()}`}],
execTime: 0,
timedOut: false,
};
if (await RemoteExecutionQuery.isPossible(execTriple)) {
fullResult.execResult = await this.runExecutableRemotely(
executablePackageHash,
executeOptions,
execTriple,
);
fullResult.didExecute = true;
} else {
fullResult.execResult = {
code: -1,
okToCache: false,
stdout: [],
stderr: [{text: `No execution available for ${execTriple.toString()}`}],
execTime: 0,
timedOut: false,
};
}
}
}
}
@@ -2910,6 +2938,12 @@ export class BaseCompiler {
if (fullResult.result) delete fullResult.result.dirPath;
// Cleanup temp directory after execution is complete
await this.doTempfolderCleanup(fullResult);
if (fullResult.result) {
await this.doTempfolderCleanup(fullResult.result);
}
this.cleanupResult(fullResult);
fullResult.s3Key = BaseCache.hash(cacheKey);
@@ -3032,7 +3066,7 @@ export class BaseCompiler {
);
if (result.execResult?.buildResult) {
this.doTempfolderCleanup(result.execResult.buildResult);
await this.doTempfolderCleanup(result.execResult.buildResult);
}
}
return result;
@@ -3049,7 +3083,7 @@ export class BaseCompiler {
if (backendOptions.executorRequest) {
const execResult = await this.handleExecution(key, executeOptions, bypassCache);
if (execResult?.buildResult) {
this.doTempfolderCleanup(execResult.buildResult);
await this.doTempfolderCleanup(execResult.buildResult);
}
return execResult;
}
@@ -3167,9 +3201,12 @@ export class BaseCompiler {
if (!backendOptions.skipPopArgs) result.popularArguments = this.possibleArguments.getPopularArguments(options);
this.doTempfolderCleanup(result);
if (result.buildResult) {
this.doTempfolderCleanup(result.buildResult);
// Only cleanup immediately if not delaying caching (e.g., not in cmake flow)
if (!delayCaching) {
await this.doTempfolderCleanup(result);
if (result.buildResult) {
await this.doTempfolderCleanup(result.buildResult);
}
}
result = this.postCompilationPreCacheHook(result);
@@ -3191,7 +3228,7 @@ export class BaseCompiler {
result.execResult = (await execPromise) as CompilationResult;
if (result.execResult.buildResult) {
this.doTempfolderCleanup(result.execResult.buildResult);
await this.doTempfolderCleanup(result.execResult.buildResult);
}
}

3
lib/compilation-queue.ts
View File

@@ -115,8 +115,9 @@ export class CompilationQueue {
status(): {busy: boolean; pending: number; size: number} {
const pending = this._queue.pending;
const size = this._queue.size;
const running = this._running.size;
return {
busy: pending > 0 || size > 0,
busy: pending > 0 || size > 0 || running > 0,
pending,
size,
};

34
lib/compilers/win32-vc.ts
View File

@@ -22,6 +22,8 @@
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
import path from 'node:path';
import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js';
import {CompilationEnvironment} from '../compilation-env.js';
import {VcAsmParser} from '../parsers/asm-parser-vc.js';
@@ -42,4 +44,36 @@ export class Win32VcCompiler extends Win32Compiler {
override getArgumentParserClass() {
return VCParser;
}
override getExtraCMakeArgs(key: any): string[] {
const args: string[] = [];
// Set CMAKE_C_COMPILER and CMAKE_CXX_COMPILER explicitly to prevent CMake from detecting MinGW
const compilerDir = path.dirname(this.compiler.exe);
const clExe = this.compiler.exe.replace(/\\/g, '/');
// For MSVC, cl.exe handles both C and C++
args.push(`-DCMAKE_C_COMPILER=${clExe}`);
args.push(`-DCMAKE_CXX_COMPILER=${clExe}`);
// Set CMAKE_LINKER, CMAKE_RC_COMPILER, and CMAKE_MT to prevent CMake from picking up MinGW tools
const linkExe = path.join(compilerDir, 'link.exe').replace(/\\/g, '/');
args.push(`-DCMAKE_LINKER=${linkExe}`);
// Try to find rc.exe and mt.exe in Windows SDK if available
if (this.compiler.includePath) {
// includePath typically contains SDK include paths
// SDK structure: Z:\compilers\windows-kits-10\bin\rc.exe
const includePathMatch = this.compiler.includePath.match(/([^;]+windows-kits-[^;]+)/i);
if (includePathMatch) {
const sdkPath = includePathMatch[1].split(/[/\\]include/i)[0];
const rcExe = path.join(sdkPath, 'bin', 'rc.exe').replace(/\\/g, '/');
const mtExe = path.join(sdkPath, 'bin', 'mt.exe').replace(/\\/g, '/');
args.push(`-DCMAKE_RC_COMPILER=${rcExe}`);
args.push(`-DCMAKE_MT=${mtExe}`);
}
}
return args;
}
}