mirrors/compiler-explorer
1
0
Fork 0
mirror of https://github.com/compiler-explorer/compiler-explorer.git synced 2025-12-27 09:23:52 -05:00
Code Issues Packages Projects Releases Wiki Activity

Seperate compiler finding process (#3022)

Browse Source
This commit is contained in:
Patrick Quist
2021-10-15 18:22:33 +02:00
committed by GitHub
parent 60217a8e61
commit 1ca884c367
7 changed files with 144 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
app.js
View File

@@ -94,6 +94,8 @@ const opts = nopt({
suppressConsoleLog: [Boolean],
metricsPort: [Number],
loki: [String],
discoveryonly: [String],
prediscovered: [String],
});
if (opts.debug) logger.level = 'debug';
@@ -464,19 +466,42 @@ async function main() {
if (gitReleaseName) logger.info(` git release ${gitReleaseName}`);
if (releaseBuildNumber) logger.info(` release build ${releaseBuildNumber}`);
const initialFindResults = await compilerFinder.find();
const initialCompilers = initialFindResults.compilers;
let initialCompilers;
let prevCompilers;
if (defArgs.ensureNoCompilerClash) {
logger.warn('Ensuring no compiler ids clash');
if (initialFindResults.foundClash) {
// If we are forced to have no clashes, throw an error with some explanation
throw new Error('Clashing compilers in the current environment found!');
} else {
logger.info('No clashing ids found, continuing normally...');
if (opts.prediscovered) {
const prediscoveredCompilersJson = await fs.readFile(opts.prediscovered);
initialCompilers = JSON.parse(prediscoveredCompilersJson);
await compilerFinder.loadPrediscovered(initialCompilers);
} else {
const initialFindResults = await compilerFinder.find();
initialCompilers = initialFindResults.compilers;
if (defArgs.ensureNoCompilerClash) {
logger.warn('Ensuring no compiler ids clash');
if (initialFindResults.foundClash) {
// If we are forced to have no clashes, throw an error with some explanation
throw new Error('Clashing compilers in the current environment found!');
} else {
logger.info('No clashing ids found, continuing normally...');
}
}
}
if (opts.discoveryonly) {
for (const compiler of initialCompilers) {
if (compiler.buildenvsetup && compiler.buildenvsetup.id === '')
delete compiler.buildenvsetup;
const compilerInstance = compilerFinder.compileHandler.findCompiler(compiler.lang, compiler.id);
if (compilerInstance) {
compiler.cachedPossibleArguments = compilerInstance.possibleArguments.possibleArguments;
}
}
await fs.writeFile(opts.discoveryonly, JSON.stringify(initialCompilers));
logger.info(`Discovered compilers saved to ${opts.discoveryonly}`);
process.exit(0);
}
const webServer = express(), router = express.Router();
setupSentry(aws.getConfig('sentryDsn'), webServer);
const healthCheckFilePath = ceProps('healthCheckFilePath', false);

6
etc/config/aws.no-s3.properties Normal file
View File

@@ -0,0 +1,6 @@
region=
storageBucket=
storagePrefix=
storageDynamoTable=
storageBucketArgStats=
storagePrefixArgStats=

4
etc/config/compiler-explorer.no-s3.properties Normal file
View File

@@ -0,0 +1,4 @@
cacheConfig=InMemory(25)
executableCacheConfig=InMemory(25)
compilerCacheConfig=InMemory(25)
storageSolution=local

64
lib/base-compiler.js
View File

@@ -1782,33 +1782,36 @@ Please select another pass or change filters.`;
clientOptions.libs[this.lang.id]);
}
async initialise(mtime, clientOptions) {
async initialise(mtime, clientOptions, isPrediscovered = false) {
this.mtime = mtime;
if (this.getRemote()) return this;
const compiler = this.compiler.exe;
const versionRe = new RegExp(this.compiler.versionRe || '.*', 'i');
const result = await this.getVersion();
if (!result) return null;
if (result.code !== 0) {
logger.warn(`Compiler '${compiler}' - non-zero result ${result.code}`);
let compiler = this.compiler.exe;
let version = this.compiler.version || '';
if (!isPrediscovered) {
const versionRe = new RegExp(this.compiler.versionRe || '.*', 'i');
const result = await this.getVersion();
if (!result) return null;
if (result.code !== 0) {
logger.warn(`Compiler '${compiler}' - non-zero result ${result.code}`);
}
let fullVersion = result.stdout + result.stderr;
_.each(utils.splitLines(fullVersion), line => {
if (version) return;
const match = line.match(versionRe);
if (match) version = match[0];
});
if (!version) {
logger.error(`Unable to find compiler version for '${compiler}' with re ${versionRe}:`, result);
return null;
}
logger.debug(`${compiler} is version '${version}'`);
this.compiler.version = version;
this.compiler.fullVersion = fullVersion;
this.compiler.supportsCfg = this.isCfgCompiler(version);
this.compiler.supportsAstView = this.couldSupportASTDump(version);
}
let version = '';
let fullVersion = result.stdout + result.stderr;
_.each(utils.splitLines(fullVersion), line => {
if (version) return;
const match = line.match(versionRe);
if (match) version = match[0];
});
if (!version) {
logger.error(`Unable to find compiler version for '${compiler}' with re ${versionRe}:`, result);
return null;
}
logger.debug(`${compiler} is version '${version}'`);
this.compiler.version = version;
this.compiler.fullVersion = fullVersion;
this.compiler.supportsCfg = this.isCfgCompiler(version);
this.compiler.supportsAstView = this.couldSupportASTDump(version);
try {
this.cmakeBaseEnv = await this.getCmakeBaseEnv();
@@ -1818,9 +1821,18 @@ Please select another pass or change filters.`;
this.initialiseLibraries(clientOptions);
const initResult = await this.getArgumentParser().parse(this);
logger.info(`${compiler} ${version} is ready`);
return initResult;
if (!isPrediscovered) {
const initResult = await this.getArgumentParser().parse(this);
logger.info(`${compiler} ${version} is ready`);
return initResult;
} else {
logger.info(`${compiler} ${version} is ready`);
if (this.compiler.cachedPossibleArguments) {
this.possibleArguments.populateOptions(this.compiler.cachedPossibleArguments);
delete this.compiler.cachedPossibleArguments;
}
return this;
}
}
getInfo() {

13
lib/compiler-finder.js
View File

@@ -369,4 +369,17 @@ export class CompilerFinder {
const result = this.ensureDistinct(_.compact(compilers));
return {foundClash: result.foundClash, compilers: _.sortBy(result.compilers, 'name')};
}
async loadPrediscovered(compilers) {
for (const compiler of compilers) {
if (compiler.buildenvsetup) {
const langId = compiler.lang;
compiler.buildenvsetup.props = (propName, def) => {
return this.compilerProps(langId, 'buildenvsetup.' + propName, def);
};
}
}
return this.compileHandler.setCompilers(compilers, this.optionsHandler.get());
}
}

46
lib/compilers/argument-parsers.js
View File

@@ -81,13 +81,7 @@ export class BaseParser {
}
export class GCCParser extends BaseParser {
static async parse(compiler) {
const results = await Promise.all([
GCCParser.getOptions(compiler, '-fsyntax-only --target-help'),
GCCParser.getOptions(compiler, '-fsyntax-only --help=common'),
GCCParser.getOptions(compiler, '-fsyntax-only --help=optimizers'),
]);
const options = Object.assign({}, ...results);
static async setCompilerSettingsFromOptions(compiler, options) {
const keys = _.keys(options);
logger.debug(`gcc-like compiler options: ${keys.join(' ')}`);
if (BaseParser.hasSupport(options, '-masm=')) {
@@ -107,6 +101,16 @@ export class GCCParser extends BaseParser {
if (_.find(keys, key => key.startsWith('-fdump-'))) {
compiler.compiler.supportsGccDump = true;
}
}
static async parse(compiler) {
const results = await Promise.all([
GCCParser.getOptions(compiler, '-fsyntax-only --target-help'),
GCCParser.getOptions(compiler, '-fsyntax-only --help=common'),
GCCParser.getOptions(compiler, '-fsyntax-only --help=optimizers'),
]);
const options = Object.assign({}, ...results);
this.setCompilerSettingsFromOptions(compiler, options);
return compiler;
}
@@ -121,8 +125,7 @@ export class GCCParser extends BaseParser {
}
export class ClangParser extends BaseParser {
static async parse(compiler) {
const options = await ClangParser.getOptions(compiler, '--help');
static async setCompilerSettingsFromOptions(compiler, options) {
logger.debug(`clang-like compiler options: ${_.keys(options).join(' ')}`);
if (BaseParser.hasSupport(options, '-fsave-optimization-record')) {
compiler.compiler.optArg = '-fsave-optimization-record';
@@ -140,6 +143,11 @@ export class ClangParser extends BaseParser {
if (compiler.compiler.options) compiler.compiler.options += ' ';
compiler.compiler.options += '-fno-crash-diagnostics';
}
}
static async parse(compiler) {
const options = await ClangParser.getOptions(compiler, '--help');
this.setCompilerSettingsFromOptions(compiler, options);
return compiler;
}
}
@@ -152,12 +160,16 @@ export class PascalParser extends BaseParser {
}
export class ISPCParser extends BaseParser {
static async parse(compiler) {
const options = await ISPCParser.getOptions(compiler, '--help');
static async setCompilerSettingsFromOptions(compiler, options) {
if (BaseParser.hasSupport(options, '--x86-asm-syntax')) {
compiler.compiler.intelAsm = '--x86-asm-syntax=intel';
compiler.compiler.supportsIntel = true;
}
}
static async parse(compiler) {
const options = await ISPCParser.getOptions(compiler, '--help');
this.setCompilerSettingsFromOptions(compiler, options);
return compiler;
}
@@ -264,6 +276,13 @@ export class VCParser extends BaseParser {
}
export class RustParser extends BaseParser {
static async setCompilerSettingsFromOptions(compiler, options) {
if (BaseParser.hasSupport(options, '--color')) {
if (compiler.compiler.options) compiler.compiler.options += ' ';
compiler.compiler.options += '--color=always';
}
}
static async parse(compiler) {
const results = await Promise.all([
RustParser.getOptions(compiler, '--help'),
@@ -271,10 +290,7 @@ export class RustParser extends BaseParser {
RustParser.getOptions(compiler, '--help -v'),
]);
const options = Object.assign({}, ...results);
if (BaseParser.hasSupport(options, '--color')) {
if (compiler.compiler.options) compiler.compiler.options += ' ';
compiler.compiler.options += '--color=always';
}
this.setCompilerSettingsFromOptions(compiler, options);
return compiler;
}

24
lib/handlers/compile.js
View File

@@ -127,6 +127,8 @@ export class CompileHandler {
}
async create(compiler) {
const isPrediscovered = !!compiler.version;
const type = compiler.compilerType || 'default';
const compilerClass = getCompilerTypeByKey(type);
@@ -145,14 +147,20 @@ export class CompileHandler {
// Try stat'ing the compiler to cache its mtime and only re-run it if it
// has changed since the last time.
try {
const res = await fs.stat(compiler.exe);
const cached = this.findCompiler(compiler.lang, compiler.id);
if (cached && cached.mtime.getTime() === res.mtime.getTime()) {
logger.debug(`${compiler.id} is unchanged`);
return cached;
let modificationTime;
if (!isPrediscovered) {
const res = await fs.stat(compiler.exe);
modificationTime = res.mtime;
const cached = this.findCompiler(compiler.lang, compiler.id);
if (cached && cached.mtime.getTime() === res.mtime.getTime()) {
logger.debug(`${compiler.id} is unchanged`);
return cached;
}
} else {
modificationTime = compiler.mtime;
}
const compilerObj = new compilerClass(compiler, this.compilerEnv);
return compilerObj.initialise(res.mtime, this.clientOptions);
return compilerObj.initialise(modificationTime, this.clientOptions, isPrediscovered);
} catch (err) {
logger.warn(`Unable to stat ${compiler.id} compiler binary: `, err);
return null;
@@ -167,12 +175,16 @@ export class CompileHandler {
const compilersById = {};
try {
this.clientOptions = clientOptions;
logger.info('Creating compilers: ' + compilers.length);
let compilersCreated = 0;
const createdCompilers = _.compact(await Promise.all(_.map(compilers, this.create, this)));
for (const compiler of createdCompilers) {
const langId = compiler.compiler.lang;
if (!compilersById[langId]) compilersById[langId] = {};
compilersById[langId][compiler.compiler.id] = compiler;
compilersCreated++;
}
logger.info('Compilers created: ' + compilersCreated);
if (this.awsProps) {
logger.info('Fetching possible arguments from storage');
await Promise.all(createdCompilers.map(