compiler-explorer/lib/utils.ts

// Copyright (c) 2016, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

import crypto from 'node:crypto';
import fs from 'node:fs/promises';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {Buffer} from 'buffer';

import {ComponentConfig, ItemConfigType} from 'golden-layout';
import semverParser from 'semver';
import _ from 'underscore';

import type {ParsedAsmResultLine} from '../types/asmresult/asmresult.interfaces.js';
import type {CacheableValue} from '../types/cache.interfaces.js';
import {BasicExecutionResult, UnprocessedExecResult} from '../types/execution/execution.interfaces.js';
import {LanguageKey} from '../types/languages.interfaces.js';
import type {Fix, ResultLine} from '../types/resultline/resultline.interfaces.js';

const tabsRe = /\t/g;
const lineRe = /\r?\n/;

export const ce_temp_prefix = 'compiler-explorer-compiler';

export function splitLines(text: string): string[] {
    if (!text) return [];
    const result = text.split(lineRe);
    if (result.length > 0 && result[result.length - 1] === '') return result.slice(0, -1);
    return result;
}

/**
 * Extracts text lines from either raw assembly string or parsed assembly objects.
 * Handles the union type safely without unsafe casting.
 */
export function extractTextLines(asm: string | ParsedAsmResultLine[]): string[] {
    if (typeof asm === 'string') {
        return splitLines(asm);
    }
    if (Array.isArray(asm)) {
        // Already parsed - extract text from each line object
        return asm.map(line => line.text);
    }
    throw new Error(`extractTextLines called with unexpected type: ${typeof asm}`);
}

/**
 * Converts assembly data to string format for parser consumption.
 * Handles the union type from CompilationInfo.asm safely.
 */
export function normalizeAsmToString(asm: string | ParsedAsmResultLine[] | undefined): string {
    if (!asm) return '';
    return typeof asm === 'string' ? asm : extractTextLines(asm).join('\n');
}

/**
 * Applies a function to each line of text split by `splitLines`
 */
export function eachLine(text: string, func: (line: string) => void): void {
    for (const line of splitLines(text)) {
        func(line);
    }
}

export function expandTabs(line: string): string {
    let extraChars = 0;
    return line.replaceAll(tabsRe, (match, offset) => {
        const total = offset + extraChars;
        const spacesNeeded = (total + 8) & 7;
        extraChars += spacesNeeded - 1;
        return '        '.substring(spacesNeeded);
    });
}

// Matches everything up to and through a CE temp dir, `.../<ce_temp_prefix><suffix>/`.
// We key off the ce_temp_prefix marker, NOT the live os.tmpdir(): the temp root varies
// by host/config (macOS /var vs /private/var, an execution.tempDirRoot elsewhere) and
// the path being masked may have been recorded under a different tmpdir than this one.
// The marker is the only invariant — it's the same constant we create the dir with.
// `(?:[A-Za-z]:)?` + `/` handles Windows too; `[^/\s]+` confines the match to one
// non-empty path token so an embedded `-I/tmp/<prefix>XXX/inc` still masks to
// `-I/app/inc` (real temp paths never have empty `//` segments).
// A user path that itself contains a `<ce_temp_prefix>...` segment would be masked too,
// but that only affects displayed output (never what's compiled/executed) and needs a
// deliberately odd dir name, so it's not worth a costlier scheme to prevent.
const TEMPDIR_RE = new RegExp(`(?:[A-Za-z]:)?/(?:[^/\\s]+/)*${ce_temp_prefix}[\\w.-]*/`);

/**
 * Removes the root dir from the given filepath, so that it will match to the user's filenames used
 *  note: will keep /app/ if instead of filepath something like '-I/tmp/path' is used
 */
export function maskRootdir(filepath: string): string {
    // TODO: this falsy guard is load-bearing against runtime `undefined` that the types
    // don't catch — not declared `string | undefined` callers (TS would reject those),
    // but type holes: a JSON.parse(...) result typed `any` (base-compiler cleanup) and a
    // Record<number,string> index that's really `undefined` when the key is missing
    // (noUncheckedIndexedAccess is off). Tighten those two sites, then this can go.
    if (!filepath) return filepath;
    // TEMPDIR_RE has a repeated path-segment group that backtracks on long input, and
    // maskRootdir runs on every output line. Gate it behind a cheap linear substring
    // check: the regex cannot match without the marker anyway. The trailing /app/ strip
    // is anchored (no backtracking) and must run regardless, as paths may already be
    // /app/-rooted from an earlier mask.
    const masked = filepath.includes(ce_temp_prefix) ? filepath.replace(TEMPDIR_RE, '/app/') : filepath;
    return masked.replace(/^\/app\//, '');
}

export function changeExtension(filename: string, newExtension: string): string {
    const lastDot = filename.lastIndexOf('.');
    if (lastDot === -1) return filename + newExtension;
    return filename.substring(0, lastDot) + newExtension;
}

const ansiColoursRe = /\x1B\[[\d;]*[Km]/g;
const terminalHyperlinkEscapeRe = /\x1B]8;;.*?(\x1B\\|\x07)(.*?)\x1B]8;;\1/g;

export function filterEscapeSequences(line: string): string {
    return line.replaceAll(ansiColoursRe, '').replaceAll(terminalHyperlinkEscapeRe, '$2');
}

function _parseOutputLine(line: string, inputFilename?: string, pathPrefix?: string) {
    line = line.split('<stdin>').join('<source>');
    if (pathPrefix) line = line.replace(pathPrefix, '');
    if (inputFilename) {
        line = line.split(inputFilename).join('<source>');

        if (inputFilename.indexOf('./') === 0) {
            line = line.split('/home/ubuntu/' + inputFilename.substring(2)).join('<source>');
            line = line.split('/home/ce/' + inputFilename.substring(2)).join('<source>');
        }
    }
    return line;
}

function parseSeverity(message: string): number {
    if (message.startsWith('warning')) return 2;
    if (message.startsWith('note')) return 1;
    return 3;
}

const SOURCE_RE = /^\s*<source>[(:](\d+)(:?,?(\d+):?)?[):]*\s*(.*)/;
const SOURCE_WITH_FILENAME = /^\s*([\w.]*\.[\w.]+)[(:](\d+)(:?,?(\d+):?)?[):]*\s*(.*)/;
const ATFILELINE_RE = /\s*at ([\w-/.]+):(\d+)/;

export enum LineParseOption {
    SourceMasking = 0,
    RootMasking = 1,
    SourceWithLineMessage = 2,
    FileWithLineMessage = 3,
    AtFileLine = 4,
}

export type LineParseOptions = LineParseOption[];

export const DefaultLineParseOptions = [
    LineParseOption.SourceMasking,
    LineParseOption.RootMasking,
    LineParseOption.SourceWithLineMessage,
    LineParseOption.FileWithLineMessage,
];

function applyParse_SourceWithLine(lineObj: ResultLine, filteredLine: string, inputFilename?: string) {
    const match = filteredLine.match(SOURCE_RE);
    if (match) {
        const message = match[4].trim();
        lineObj.tag = {
            line: Number.parseInt(match[1], 10),
            column: Number.parseInt(match[3] || '0', 10),
            text: message,
            severity: parseSeverity(message),
            file: inputFilename ? path.basename(inputFilename) : undefined,
        };
    }
}

function applyParse_FileWithLine(lineObj: ResultLine, filteredLine: string) {
    const match = filteredLine.match(SOURCE_WITH_FILENAME);
    if (match) {
        const filename = match[1];
        const message = match[5].trim();
        lineObj.tag = {
            file: filename,
            line: Number.parseInt(match[2], 10),
            column: Number.parseInt(match[4] || '0', 10),
            text: message,
            severity: parseSeverity(message),
        };
    }
}

function applyParse_AtFileLine(lineObj: ResultLine, filteredLine: string) {
    const match = filteredLine.match(ATFILELINE_RE);
    if (match) {
        if (match[1].startsWith('/app/')) {
            lineObj.tag = {
                file: match[1].replace(/^\/app\//, ''),
                line: Number.parseInt(match[2], 10),
                column: 0,
                text: filteredLine,
                severity: 3,
            };
        } else if (!match[1].startsWith('/')) {
            lineObj.tag = {
                file: match[1],
                line: Number.parseInt(match[2], 10),
                column: 0,
                text: filteredLine,
                severity: 3,
            };
        }
    }
}

export function parseOutput(
    lines: string,
    inputFilename?: string,
    pathPrefix?: string,
    options: LineParseOptions = DefaultLineParseOptions,
): ResultLine[] {
    const result: ResultLine[] = [];
    eachLine(lines, line => {
        if (options.includes(LineParseOption.SourceMasking)) {
            line = _parseOutputLine(line, inputFilename, pathPrefix);
        }
        if (!inputFilename && options.includes(LineParseOption.RootMasking)) {
            line = maskRootdir(line);
        }
        if (line !== null) {
            const lineObj: ResultLine = {text: line};
            const filteredLine = filterEscapeSequences(line);

            if (options.includes(LineParseOption.SourceWithLineMessage))
                applyParse_SourceWithLine(lineObj, filteredLine, inputFilename);

            if (!lineObj.tag && options.includes(LineParseOption.FileWithLineMessage))
                applyParse_FileWithLine(lineObj, filteredLine);

            if (!lineObj.tag && options.includes(LineParseOption.AtFileLine))
                applyParse_AtFileLine(lineObj, filteredLine);

            result.push(lineObj);
        }
    });
    return result;
}

export function parseRustOutput(lines: string, inputFilename?: string, pathPrefix?: string) {
    const inputBasename = inputFilename ? path.basename(inputFilename) : undefined;
    const quickfixes: {re: RegExp; makeFix: (match: string[]) => Fix}[] = [
        {
            re: / *help: add `#!\[feature\((.*?)\)]`/,
            makeFix: ([_, featureName]) => ({
                title: `Add feature flag \`${featureName}\``,
                edits: [
                    {
                        line: 1,
                        column: 1,
                        endline: 1,
                        endcolumn: 1,
                        text: `#![feature(${featureName})]\n`,
                    },
                ],
            }),
        },
        {
            re: / *(\d+) *\+ ( *)use (.*?);$/,
            makeFix: ([_, line, indentation, path]) => ({
                title: `Add import for \`${path}\``,
                edits: [
                    {
                        line: Number.parseInt(line, 10),
                        column: 1,
                        endline: Number.parseInt(line, 10),
                        endcolumn: 1,
                        text: `${indentation}use ${path};\n`,
                    },
                ],
            }),
        },
    ];

    const re = /^\s+-->\s+(?<filename>.*):(?<line>\d+):(?<column>\d+)/;
    const result: ResultLine[] = [];
    let currentDiagnostic: ResultLine | undefined;
    eachLine(lines, line => {
        line = _parseOutputLine(line, inputFilename, pathPrefix);
        if (line !== null) {
            const lineObj: ResultLine = {text: line};
            const filteredLine = filterEscapeSequences(line);
            const match = filteredLine.match(re);

            if (match?.groups) {
                const file =
                    match.groups.filename === '<source>' ? inputBasename : path.basename(match.groups.filename);
                const line = Number.parseInt(match.groups.line, 10);
                const column = Number.parseInt(match.groups.column, 10);

                currentDiagnostic = result.pop();
                if (currentDiagnostic !== undefined) {
                    const text = filterEscapeSequences(currentDiagnostic.text);
                    currentDiagnostic.tag = {
                        file,
                        line,
                        column,
                        text,
                        severity: parseSeverity(text),
                    };
                    result.push(currentDiagnostic);
                }

                lineObj.tag = {
                    file,
                    line,
                    column,
                    text: '', // Left empty so that it does not show up in the editor
                    severity: 3,
                };
            }

            const fixes = quickfixes.flatMap(({re, makeFix}) => {
                const match = filteredLine.match(re);
                if (match) {
                    return [makeFix(match)];
                } else {
                    return [];
                }
            });

            if (currentDiagnostic?.tag && fixes.length !== 0) {
                lineObj.tag = {...currentDiagnostic.tag, fixes};
            }

            result.push(lineObj);
        }
    });
    return result;
}

/***
 * Anonymizes given IP.
 * For IPv4, it removes the last octet
 * For IPv6, it removes the last three hextets
 *
 * @param {string} ip - IP string, of either type (localhost|IPv4|IPv6)
 * @returns {string} Anonymized IP
 */
export function anonymizeIp(ip: string): string {
    if (ip.includes('localhost')) {
        return ip;
    }
    if (ip.includes(':')) {
        // IPv6
        return ip.replace(/(?::[\dA-Fa-f]{0,4}){3}$/, ':0:0:0');
    }
    // IPv4
    return ip.replace(/\.\d{1,3}$/, '.0');
}

/***
 *
 * @param {*} object
 * @returns {string}
 */
function objectToHashableString(object: CacheableValue): string {
    // See https://stackoverflow.com/questions/899574/which-is-best-to-use-typeof-or-instanceof/6625960#6625960
    return typeof object === 'string' ? object : JSON.stringify(object);
}

const DefaultHash = 'Compiler Explorer Default Version 1';

/***
 * Gets the hash (as a binary buffer) of the given object
 *
 * Limitation: object shall not have circular references
 * @param {*} object - Object to get hash from
 * @param {string} [HashVersion=DefaultHash] - Hash "version" key
 * @returns {Buffer} - Hash of object
 */
export function getBinaryHash(object: CacheableValue, HashVersion = DefaultHash): Buffer {
    return crypto.createHmac('sha256', HashVersion).update(objectToHashableString(object)).digest();
}

/***
 * Gets the hash (as a hex string) of the given object
 *
 * Limitation: object shall not have circular references
 * @param {*} object - Object to get hash from
 * @param {string} [HashVersion=DefaultHash] - Hash "version" key
 * @returns {string} - Hash of object
 */
export function getHash(object: CacheableValue, HashVersion = DefaultHash): string {
    return crypto.createHmac('sha256', HashVersion).update(objectToHashableString(object)).digest('hex');
}

interface glEditorMainContent {
    // Editor content
    source: string;
    // Editor syntax language
    language: LanguageKey;
}

interface glCompilerMainContent {
    // Compiler id
    compiler: string;
}

interface glContents {
    editors: glEditorMainContent[];
    compilers: glCompilerMainContent[];
}

/***
 * Gets every (source, lang) & (compilerId) available
 * @param {Array} content - GoldenLayout config topmost content field
 * @returns {glContents}
 */
export function glGetMainContents(content: ItemConfigType[] = []): glContents {
    const contents: glContents = {editors: [], compilers: []};
    _.each(content, element => {
        if (element.type === 'component') {
            const component = element as ComponentConfig;
            if (component.componentName === 'codeEditor') {
                contents.editors.push({
                    source: component.componentState.source,
                    language: component.componentState.lang as LanguageKey,
                });
            } else if (component.componentName === 'compiler') {
                contents.compilers.push({
                    compiler: component.componentState.compiler,
                });
            }
        } else {
            const subComponents = glGetMainContents(element.content);
            contents.editors = contents.editors.concat(subComponents.editors);
            contents.compilers = contents.compilers.concat(subComponents.compilers);
        }
    });
    return contents;
}

export function squashHorizontalWhitespace(line: string, atStart = true): string {
    if (line.trim().length === 0) {
        return '';
    }
    const splat = line.split(/\s+/);
    if (splat[0] === '' && atStart) {
        // An indented line: preserve a two-space indent (max)
        const intent = line[1] === ' ' ? '  ' : ' ';
        return intent + splat.slice(1).join(' ');
    }
    return splat.join(' ');
}

export function toProperty(prop: string): boolean | number | string {
    if (prop === 'true' || prop === 'yes') return true;
    if (prop === 'false' || prop === 'no') return false;
    if (/^-?(0|[1-9]\d*)$/.test(prop)) return Number.parseInt(prop, 10);
    if (/^-?\d*\.\d+$/.test(prop)) return Number.parseFloat(prop);
    return prop;
}

// Initially based on http://philzimmermann.com/docs/human-oriented-base-32-encoding.txt
const BASE32_ALPHABET = '13456789EGKMPTWYabcdefhjnoqrsvxz';

/***
 *  A Base32 encoding implementation valid for our needs.
 *  Of importance, note that there's no padding
 * @param {Buffer} buffer
 * @returns {string}
 */
export function base32Encode(buffer: Buffer): string {
    let output = '';
    // This can grow up to 12 bits
    let digest = 0;
    // How many bits are we actually using from digest
    let bits = 0;

    function encodeNewWord() {
        // Get first 5 bits
        const word = digest & 0b11111;
        const character = BASE32_ALPHABET[word];
        output += character;
        bits -= 5;
        // Shift out the newly processed word
        digest = digest >>> 5;
    }

    for (const byte of buffer) {
        // Append current byte to digest
        digest = (byte << bits) | digest;
        bits += 8;

        // Add characters until we run out of bits
        while (bits >= 5) {
            encodeNewWord();
        }
    }
    // Do we need a last pass?
    if (bits !== 0) {
        encodeNewWord();
    }

    return output;
}

// Splits a : separated list into its own array, or to default if input is undefined
export function splitIntoArray(input?: string, defaultArray: string[] = []): string[] {
    if (input === undefined) {
        return defaultArray;
    }
    return input.split(':');
}

/***
 * Absolute path to the root of the application
 */
export const APP_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');

export function resolvePathFromAppRoot(...args: string[]) {
    return path.resolve(APP_ROOT, ...args);
}

export async function fileExists(filename: string): Promise<boolean> {
    try {
        const stat = await fs.stat(filename);
        return stat.isFile();
    } catch {
        return false;
    }
}

export async function dirExists(dir: string): Promise<boolean> {
    try {
        const stat = await fs.stat(dir);
        return stat.isDirectory();
    } catch {
        return false;
    }
}

export function countOccurrences<T>(collection: Iterable<T>, item: T): number {
    // _.reduce(collection, (total, value) => value === item ? total + 1 : total, 0) would work, but is probably slower
    let result = 0;
    for (const element of collection) {
        if (element === item) {
            result++;
        }
    }
    return result;
}

export enum magic_semver {
    trunk = '99999999.99999.999',
    non_trunk = '99999998.99999.999',
}

export function asSafeVer(semver: string | number | null | undefined): string {
    if (semver != null) {
        if (typeof semver === 'number') {
            semver = `${semver}`;
        }
        const splits = semver.split(' ');
        if (splits.length > 0) {
            let interestingPart = splits[0];
            let dotCount = countOccurrences(interestingPart, '.');
            for (; dotCount < 2; dotCount++) {
                interestingPart += '.0';
            }
            const validated: string | null = semverParser.valid(interestingPart, true);
            if (validated != null) {
                return validated;
            }
        }

        if (semver.includes('trunk') || semver.includes('main')) {
            return magic_semver.trunk;
        }
    }
    return magic_semver.non_trunk;
}

export function processExecutionResult(input: UnprocessedExecResult, inputFilename?: string): BasicExecutionResult {
    const start = performance.now();
    const stdout = parseOutput(input.stdout, inputFilename);
    const stderr = parseOutput(input.stderr, inputFilename);
    const end = performance.now();
    return {
        ...input,
        stdout,
        stderr,
        processExecutionResultTime: end - start,
    };
}

export function getEmptyExecutionResult(): BasicExecutionResult {
    return {
        code: -1,
        okToCache: false,
        filenameTransform: x => x,
        stdout: [],
        stderr: [],
        execTime: 0,
        timedOut: false,
    };
}

export function deltaTimeNanoToMili(startTime: bigint, endTime: bigint): number {
    return Number((endTime - startTime) / BigInt(1_000_000));
}

/**
 * Sleep for a number of milliseconds.
 */
export async function sleep(ms: number) {
    return new Promise(resolve => setTimeout(resolve, ms));
}

export function resultLinesToText(lines: ResultLine[]): string {
    return lines.map(line => line.text).join('\n');
}

/**
 * Try and read a file as a utf-8 text file, returning its contents if present, or undefined if not.
 */
export async function tryReadTextFile(filename: string): Promise<string | undefined> {
    try {
        return await fs.readFile(filename, 'utf8');
    } catch {
        return undefined;
    }
}

/**
 * Try and read a file as a utf-8 json file, returning its contents if present, or undefined if not.
 */
export async function tryReadJsonFile(filename: string): Promise<any | undefined> {
    const text = await tryReadTextFile(filename);
    return text === undefined ? undefined : JSON.parse(text);
}

/**
 * Output a file, creating any necessary directories.
 */
export async function outputTextFile(filepath: string, contents: string): Promise<void> {
    await fs.mkdir(path.dirname(filepath), {recursive: true});
    await fs.writeFile(filepath, contents, 'utf-8');
}

/**
 * Create an empty file (and directories leading to it), leaving it alone if already present.
 */
export async function ensureFileExists(filepath: string): Promise<void> {
    await fs.mkdir(path.dirname(filepath), {recursive: true});
    await (await fs.open(filepath, 'a')).close();
}