mirror of
https://github.com/compiler-explorer/compiler-explorer.git
synced 2026-07-16 17:26:58 -04:00
9db0c4d28b
Part of #8811 (production disk-full incident). Companion to #8812 — this is the systemic backstop so no future never-settling job can wedge the system. Two changes: **Queue timeout was dead config.** `enqueue()` passed `timeout: undefined` to p-queue's `add()`; p-queue spreads per-call options over its defaults, so this overrode and *disabled* the queue-wide timeout configured from `compilationEnvTimeoutMs` (default 300s). A job whose promise never settled therefore occupied a queue slot forever — and since temp dir cleanup only runs when the queue is fully idle, one wedged slot permanently disabled cleanup and filled the disk. Removing the override makes the timeout effective: in p-queue v9 a timed-out task rejects with `TimeoutError`, freeing the slot. (The timeout doesn't kill underlying work — that remains the exec layer's job — but the system makes progress again.) **Uncaught exceptions now actually stop the process.** The handler set `process.exitCode = 1` assuming the app would "exit naturally", but a process with live server listeners never does: during the incident the instance limped on half-dead for hours, passing healthchecks while every compilation failed. Now it exits after a 1s delay (letting winston flush its transports), and the load balancer replaces the instance. The new queue test fails against the previous code: a never-settling job is never rejected and `status().busy` stays true forever. Also adds basic enqueue/nested-enqueue coverage. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: mattgodbolt-molty <mattgodbolt-molty@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
89 lines
3.7 KiB
TypeScript
Executable File
89 lines
3.7 KiB
TypeScript
Executable File
// Copyright (c) 2025, Compiler Explorer Authors
|
|
// All rights reserved.
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions are met:
|
|
//
|
|
// * Redistributions of source code must retain the above copyright notice,
|
|
// this list of conditions and the following disclaimer.
|
|
// * Redistributions in binary form must reproduce the above copyright
|
|
// notice, this list of conditions and the following disclaimer in the
|
|
// documentation and/or other materials provided with the distribution.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
// POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
////
|
|
// see https://docs.sentry.io/platforms/javascript/guides/node/install/late-initialization/
|
|
import '@sentry/node/preload'; // preload Sentry's "preload" support before any other imports
|
|
////
|
|
import process from 'node:process';
|
|
|
|
import {parseArgsToAppArguments} from './lib/app/cli.js';
|
|
import {loadConfiguration} from './lib/app/config.js';
|
|
import {initialiseApplication} from './lib/app/main.js';
|
|
import {setBaseDirectory} from './lib/assert.js';
|
|
import {initialiseLogging, logger} from './lib/logger.js';
|
|
import * as props from './lib/properties.js';
|
|
import * as utils from './lib/utils.js';
|
|
|
|
// Set base directory for resolving paths
|
|
setBaseDirectory(new URL('.', import.meta.url));
|
|
|
|
// Set up signal handlers
|
|
process.on('uncaughtException', uncaughtHandler);
|
|
process.on('SIGINT', signalHandler('SIGINT'));
|
|
process.on('SIGTERM', signalHandler('SIGTERM'));
|
|
process.on('SIGQUIT', signalHandler('SIGQUIT'));
|
|
|
|
function signalHandler(name: string) {
|
|
return () => {
|
|
logger.info(`stopping process: ${name}`);
|
|
process.exit(0);
|
|
};
|
|
}
|
|
|
|
function uncaughtHandler(err: Error, origin: NodeJS.UncaughtExceptionOrigin) {
|
|
logger.error(`stopping process: Uncaught exception (origin: ${origin}):`, err);
|
|
// A process with live server listeners never exits "naturally": after an uncaught exception we'd
|
|
// limp on in an undefined state (#8811). Exit after a short delay so winston can flush its
|
|
// transports (calling `process.exit()` immediately may lose log lines, see
|
|
// https://github.com/winstonjs/winston/issues/1504#issuecomment-1033087411), and let the load
|
|
// balancer replace the instance.
|
|
process.exitCode = 1;
|
|
setTimeout(() => process.exit(1), 1000);
|
|
}
|
|
|
|
// Parse command line arguments
|
|
const appArgs = parseArgsToAppArguments(process.argv);
|
|
if (appArgs.isWsl) process.env.wsl = 'true';
|
|
|
|
// Initialize logging reasonably early in startup
|
|
initialiseLogging(appArgs.loggingOptions);
|
|
|
|
// Load configuration
|
|
const distPath = utils.resolvePathFromAppRoot('.');
|
|
const config = loadConfiguration(appArgs);
|
|
const awsProps = props.propsFor('aws');
|
|
|
|
// Initialize and start the application
|
|
initialiseApplication({
|
|
appArgs,
|
|
config,
|
|
distPath,
|
|
awsProps,
|
|
}).catch(err => {
|
|
logger.error('Top-level error (shutting down):', err);
|
|
// Shut down after a second to hopefully let logs flush.
|
|
setTimeout(() => process.exit(1), 1000);
|
|
});
|