Add library support for Go compiler (#8397)

.claude/commands/resetprops.md

@@ -0,0 +1,9 @@
+Reset all `.local.properties` files in `etc/config/` by overwriting them with the contents of their corresponding `.amazon.properties` files.
+
+Steps:
+1. Find all `*.local.properties` files in `etc/config/`
+2. For each file, determine the base name (e.g. `c++.local.properties` -> `c++`)
+3. Check if a corresponding `*.amazon.properties` file exists (e.g. `c++.amazon.properties`)
+4. If it exists, overwrite the `.local.properties` file with the contents of the `.amazon.properties` file
+5. If no corresponding `.amazon.properties` file exists, skip it and note that it was skipped
+6. Report a summary of which files were overwritten and which were skipped

etc/config/go.amazon.properties

@@ -107,6 +107,8 @@ group.gl.compilers=&x86gl:&armgl:&mipsgl:&ppcgl:&riscvgl:&s390xgl:&wasmgl
 group.gl.versionFlag=version
 group.gl.compilerType=golang
 group.gl.isSemVer=true
+group.gl.buildenvsetup=ceconan-go
+group.gl.buildenvsetup.host=https://conan.compiler-explorer.com
 
 ###### x86 GC ######
 group.x86gl.compilers=&amd64gl:&386gl
@@ -1589,7 +1591,23 @@ compiler.gppc64g9.semver=AT13.0
 #################################
 #################################
 # Installed libs (See c++.amazon.properties for a scheme of libs group)
-libs=
+
+libs=uuid:protobuf:errors
+
+libs.uuid.name=github.com/google/uuid
+libs.uuid.url=https://github.com/google/uuid
+libs.uuid.versions=v160
+libs.uuid.versions.v160.version=v1.6.0
+
+libs.protobuf.name=google.golang.org/protobuf
+libs.protobuf.url=https://github.com/protocolbuffers/protobuf-go
+libs.protobuf.versions=v1360
+libs.protobuf.versions.v1360.version=v1.36.0
+
+libs.errors.name=github.com/pkg/errors
+libs.errors.url=https://github.com/pkg/errors
+libs.errors.versions=v091
+libs.errors.versions.v091.version=v0.9.1
 
 #################################
 #################################

lib/buildenvsetup/_all.ts

@@ -25,4 +25,5 @@
 export {BuildEnvSetupCeConanDirect} from './ceconan.js';
 export {BuildEnvSetupCeConanCircleDirect} from './ceconan-circle.js';
 export {BuildEnvSetupCeConanFortranDirect} from './ceconan-fortran.js';
+export {BuildEnvSetupCeConanGoDirect} from './ceconan-go.js';
 export {BuildEnvSetupCeConanRustDirect} from './ceconan-rust.js';

lib/buildenvsetup/ceconan-go.ts

@@ -0,0 +1,99 @@
+// Copyright (c) 2025, Compiler Explorer Authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+import _ from 'underscore';
+
+import {CacheKey} from '../../types/compilation/compilation.interfaces.js';
+import {CompilerInfo} from '../../types/compiler.interfaces.js';
+import {CompilationEnvironment} from '../compilation-env.js';
+import {VersionInfo} from '../options-handler.js';
+import {ExecCompilerCachedFunc} from './base.js';
+import type {BuildEnvDownloadInfo} from './buildenv.interfaces.js';
+import {BuildEnvSetupCeConanDirect} from './ceconan.js';
+
+export class BuildEnvSetupCeConanGoDirect extends BuildEnvSetupCeConanDirect {
+    static override get key() {
+        return 'ceconan-go';
+    }
+
+    constructor(compilerInfo: CompilerInfo, env: CompilationEnvironment) {
+        super(compilerInfo, env);
+
+        this.onlyonstaticliblink = false;
+        this.extractAllToRoot = false;
+    }
+
+    override async initialise(execCompilerCachedFunc: ExecCompilerCachedFunc) {
+        if (this.compilerArch) return;
+        this.compilerSupportsX86 = true;
+    }
+
+    override getLibcxx(key: CacheKey) {
+        return '';
+    }
+
+    override getTarget(key: CacheKey) {
+        const goarch = (this.compiler.goarch || 'amd64').toString();
+        return BuildEnvSetupCeConanGoDirect.goArchToConanArch(goarch);
+    }
+
+    static goArchToConanArch(goarch: string): string {
+        switch (goarch) {
+            case 'amd64':
+                return 'x86_64';
+            case '386':
+                return 'x86';
+            case 'arm64':
+                return 'aarch64';
+            default:
+                return goarch;
+        }
+    }
+
+    override hasBinariesToLink(details: VersionInfo) {
+        return true;
+    }
+
+    override shouldDownloadPackage(details: VersionInfo) {
+        return true;
+    }
+
+    override async download(
+        key: CacheKey,
+        dirPath: string,
+        libraryDetails: Record<string, VersionInfo>,
+    ): Promise<BuildEnvDownloadInfo[]> {
+        const modifiedLibraryDetails: Record<string, VersionInfo> = {};
+
+        _.each(libraryDetails, (details: VersionInfo, libId: string) => {
+            const modifiedDetails = {...details};
+            if (!modifiedDetails.lookupname) {
+                modifiedDetails.lookupname = `go_${libId}`;
+            }
+            modifiedLibraryDetails[libId] = modifiedDetails;
+        });
+
+        return super.download(key, dirPath, modifiedLibraryDetails);
+    }
+}

lib/compilers/golang.ts

@@ -27,16 +27,25 @@ import path from 'node:path';
 
 import _ from 'underscore';
 
-import type {ExecutionOptionsWithEnv} from '../../types/compilation/compilation.interfaces.js';
+import type {CacheKey, ExecutionOptionsWithEnv} from '../../types/compilation/compilation.interfaces.js';
 import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js';
 import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js';
 import type {ResultLine} from '../../types/resultline/resultline.interfaces.js';
 import {unwrap} from '../assert.js';
 import {BaseCompiler} from '../base-compiler.js';
+import type {BuildEnvDownloadInfo} from '../buildenvsetup/buildenv.interfaces.js';
 import {CompilationEnvironment} from '../compilation-env.js';
+import {logger} from '../logger.js';
 import * as utils from '../utils.js';
 import {GolangParser} from './argument-parsers.js';
 
+interface GoLibraryMetadata {
+    module: string;
+    version: string;
+    go_mod_require: string;
+    go_sum: string;
+}
+
 // Each arch has a list of jump instructions in
 // Go source src/cmd/asm/internal/arch.
 // x86 -> j, b
@@ -57,6 +66,8 @@ type GoEnv = {
 
 export class GolangCompiler extends BaseCompiler {
     private readonly GOENV: GoEnv;
+    private hasLibraries = false;
+    private verboseBuild = false;
 
     static get key() {
         return 'golang';
@@ -107,6 +118,137 @@ export class GolangCompiler extends BaseCompiler {
         return undefined;
     }
 
+    override async setupBuildEnvironment(
+        key: CacheKey,
+        dirPath: string,
+        binary: boolean,
+    ): Promise<BuildEnvDownloadInfo[]> {
+        this.hasLibraries = key.libraries && key.libraries.length > 0;
+        return super.setupBuildEnvironment(key, dirPath, binary);
+    }
+
+    protected async findDownloadedLibraries(dirPath: string): Promise<string[]> {
+        const libraries: string[] = [];
+        try {
+            const entries = await fs.readdir(dirPath, {withFileTypes: true});
+            for (const entry of entries) {
+                if (entry.isDirectory()) {
+                    const metadataPath = path.join(dirPath, entry.name, 'metadata.json');
+                    if (await utils.fileExists(metadataPath)) {
+                        libraries.push(entry.name);
+                    }
+                }
+            }
+        } catch {
+            // Directory doesn't exist or can't be read
+        }
+        return libraries;
+    }
+
+    protected async readLibraryMetadata(dirPath: string, libId: string): Promise<GoLibraryMetadata | null> {
+        try {
+            const metadataPath = path.join(dirPath, libId, 'metadata.json');
+            const content = await fs.readFile(metadataPath, 'utf-8');
+            return JSON.parse(content) as GoLibraryMetadata;
+        } catch (e) {
+            logger.warn(`Failed to read metadata for Go library ${libId}: ${e}`);
+            return null;
+        }
+    }
+
+    protected async mergeGocache(cachePath: string, libCacheDeltaPath: string): Promise<void> {
+        if (!(await utils.dirExists(libCacheDeltaPath))) return;
+        await fs.cp(libCacheDeltaPath, cachePath, {recursive: true, force: false});
+    }
+
+    protected async setupModuleSources(goPath: string, libPath: string): Promise<void> {
+        const moduleSourcesPath = path.join(libPath, 'module_sources');
+        if (!(await utils.dirExists(moduleSourcesPath))) return;
+
+        const pkgModPath = path.join(goPath, 'pkg', 'mod');
+        await fs.mkdir(pkgModPath, {recursive: true});
+        await fs.cp(moduleSourcesPath, pkgModPath, {recursive: true, force: false});
+    }
+
+    protected async generateGoMod(inputDir: string, libraries: string[], dirPath: string): Promise<void> {
+        const goModPath = path.join(inputDir, 'go.mod');
+        const goSumPath = path.join(inputDir, 'go.sum');
+
+        let goModContent = '';
+        let goSumContent = '';
+
+        // Check if user provided their own go.mod
+        const existingGoMod = await utils.fileExists(goModPath);
+        if (existingGoMod) {
+            goModContent = await fs.readFile(goModPath, 'utf-8');
+        } else {
+            goModContent = 'module example\n\ngo 1.21\n';
+        }
+
+        // Collect require statements and sum entries from all libraries
+        const requireStatements: string[] = [];
+        const sumEntries: string[] = [];
+
+        for (const libId of libraries) {
+            const metadata = await this.readLibraryMetadata(dirPath, libId);
+            if (metadata) {
+                if (metadata.go_mod_require) {
+                    requireStatements.push(metadata.go_mod_require);
+                }
+                if (metadata.go_sum) {
+                    sumEntries.push(metadata.go_sum);
+                }
+            }
+        }
+
+        // Append require statements to go.mod
+        if (requireStatements.length > 0) {
+            if (!goModContent.includes('require (')) {
+                goModContent += '\nrequire (\n';
+                goModContent += requireStatements.map(r => `\t${r}`).join('\n');
+                goModContent += '\n)\n';
+            } else {
+                // Insert before the closing paren of require block
+                goModContent = goModContent.replace(
+                    /require \(([^)]*)\)/,
+                    (match, inner) => `require (${inner}\n${requireStatements.map(r => `\t${r}`).join('\n')}\n)`,
+                );
+            }
+        }
+
+        await fs.writeFile(goModPath, goModContent);
+
+        // Write go.sum
+        if (sumEntries.length > 0) {
+            goSumContent = sumEntries.join('\n') + '\n';
+            await fs.writeFile(goSumPath, goSumContent);
+        }
+    }
+
+    protected async setupGoLibraries(
+        inputDir: string,
+        cachePath: string,
+        goPath: string,
+        dirPath: string,
+    ): Promise<void> {
+        const libraries = await this.findDownloadedLibraries(dirPath);
+        if (libraries.length === 0) return;
+
+        for (const libId of libraries) {
+            const libPath = path.join(dirPath, libId);
+
+            // Merge cache_delta into GOCACHE
+            const cacheDeltaPath = path.join(libPath, 'cache_delta');
+            await this.mergeGocache(cachePath, cacheDeltaPath);
+
+            // Copy module_sources to GOPATH/pkg/mod
+            await this.setupModuleSources(goPath, libPath);
+        }
+
+        // Generate go.mod and go.sum
+        await this.generateGoMod(inputDir, libraries, dirPath);
+    }
+
     override async runCompiler(
         compiler: string,
         options: string[],
@@ -119,24 +261,43 @@ export class GolangCompiler extends BaseCompiler {
         }
 
         const inputDir = path.dirname(inputFilename);
+        const dirPath = inputDir;
         const tempCachePath = path.join(inputDir, 'cache');
+        const goPath = path.join(inputDir, 'gopath');
 
         execOptions.env = {
             ...execOptions.env,
             GOCACHE: tempCachePath,
+            GOPATH: goPath,
         };
 
+        this.verboseBuild = options.some(opt => opt === '-v' || opt === '-x');
+
+        // Force offline compilation when libraries are selected
+        if (this.hasLibraries) {
+            execOptions.env.GOPROXY = 'off';
+        }
+
         const sourceCachePath = await this.getSourceCachePath();
         if (sourceCachePath) {
             try {
                 await fs.mkdir(tempCachePath, {recursive: true});
-
                 await fs.cp(sourceCachePath, tempCachePath, {recursive: true, force: false});
             } catch {
                 // Cache setup failed, continue without cache
             }
         }
 
+        // Set up Go libraries if any were downloaded
+        if (this.hasLibraries) {
+            try {
+                await fs.mkdir(goPath, {recursive: true});
+                await this.setupGoLibraries(inputDir, tempCachePath, goPath, dirPath);
+            } catch (e) {
+                logger.warn(`Failed to set up Go libraries: ${e}`);
+            }
+        }
+
         return super.runCompiler(compiler, options, inputFilename, execOptions, filters);
     }
 
@@ -277,7 +438,9 @@ export class GolangCompiler extends BaseCompiler {
         }
         const logging = this.extractLogging(out);
         result.asm = this.convertNewGoL(out);
-        result.stderr = [];
+        result.stderr = this.verboseBuild
+            ? out.filter(obj => !obj.text.match(LINE_RE) && !obj.text.match(UNKNOWN_RE) && !obj.text.match(LOGGING_RE))
+            : [];
         result.stdout = utils.parseOutput(logging, result.inputFilename);
         return Promise.all([result, [], []]);
     }
@@ -286,6 +449,11 @@ export class GolangCompiler extends BaseCompiler {
         return [];
     }
 
+    override getIncludeArguments(libraries: object[]): string[] {
+        // Go uses the module system, not include flags
+        return [];
+    }
+
     override optionsForFilter(filters: ParseFiltersAndOutputOptions, outputFilename: string, userOptions?: string[]) {
         // If we're dealing with an older version...
         if (this.compiler.id === '6g141') {
@@ -293,10 +461,10 @@ export class GolangCompiler extends BaseCompiler {
         }
 
         if (filters.binary) {
-            return ['build', '-o', outputFilename, '-gcflags=' + unwrap(userOptions).join(' ')];
+            return ['build', '-trimpath', '-o', outputFilename, '-gcflags=' + unwrap(userOptions).join(' ')];
         }
         // Add userOptions to -gcflags to preserve previous behavior.
-        return ['build', '-o', outputFilename, '-gcflags=-S ' + unwrap(userOptions).join(' ')];
+        return ['build', '-trimpath', '-o', outputFilename, '-gcflags=-S ' + unwrap(userOptions).join(' ')];
     }
 
     override filterUserOptions(userOptions: string[]) {

test/golang-tests.ts

@@ -23,8 +23,11 @@
 // POSSIBILITY OF SUCH DAMAGE.
 
 import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
 
-import {beforeAll, describe, expect, it} from 'vitest';
+import {afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
 
 import {CompilationEnvironment} from '../lib/compilation-env.js';
 import {GolangCompiler} from '../lib/compilers/golang.js';
@@ -123,3 +126,169 @@ describe('GO environment variables', () => {
         expect(execOptions.env.GOCACHE).toBeUndefined();
     });
 });
+
+describe('GO library support', () => {
+    let tempDir: string;
+    let compiler: GolangCompiler;
+
+    beforeAll(() => {
+        ce = makeCompilationEnvironment({languages});
+    });
+
+    beforeEach(async () => {
+        tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'ce-go-test-'));
+        const compilerInfo = makeFakeCompilerInfo({
+            exe: '/opt/compiler-explorer/go1.20/bin/go',
+            lang: languages.go.id,
+        });
+        compiler = new GolangCompiler(compilerInfo, ce);
+    });
+
+    afterEach(async () => {
+        await fsp.rm(tempDir, {recursive: true, force: true});
+        vi.restoreAllMocks();
+    });
+
+    it('Returns empty array for getIncludeArguments', () => {
+        const result = compiler.getIncludeArguments([{paths: ['/some/path']}]);
+        expect(result).toEqual([]);
+    });
+
+    it('Finds downloaded libraries with metadata.json', async () => {
+        // Create mock library directories
+        const libDir = path.join(tempDir, 'uuid');
+        await fsp.mkdir(libDir, {recursive: true});
+        await fsp.writeFile(
+            path.join(libDir, 'metadata.json'),
+            JSON.stringify({
+                module: 'github.com/google/uuid',
+                version: 'v1.6.0',
+                go_mod_require: 'github.com/google/uuid v1.6.0',
+                go_sum: 'github.com/google/uuid v1.6.0 h1:abc123=',
+            }),
+        );
+
+        // Also create a non-library directory
+        const nonLibDir = path.join(tempDir, 'cache');
+        await fsp.mkdir(nonLibDir, {recursive: true});
+
+        const libraries = await (compiler as any).findDownloadedLibraries(tempDir);
+        expect(libraries).toEqual(['uuid']);
+    });
+
+    it('Reads library metadata correctly', async () => {
+        const libDir = path.join(tempDir, 'uuid');
+        await fsp.mkdir(libDir, {recursive: true});
+        const metadata = {
+            module: 'github.com/google/uuid',
+            version: 'v1.6.0',
+            go_mod_require: 'github.com/google/uuid v1.6.0',
+            go_sum: 'github.com/google/uuid v1.6.0 h1:abc123=',
+        };
+        await fsp.writeFile(path.join(libDir, 'metadata.json'), JSON.stringify(metadata));
+
+        const result = await (compiler as any).readLibraryMetadata(tempDir, 'uuid');
+        expect(result).toEqual(metadata);
+    });
+
+    it('Returns null for missing metadata', async () => {
+        const result = await (compiler as any).readLibraryMetadata(tempDir, 'nonexistent');
+        expect(result).toBeNull();
+    });
+
+    it('Generates go.mod with require statements', async () => {
+        const libDir = path.join(tempDir, 'uuid');
+        await fsp.mkdir(libDir, {recursive: true});
+        await fsp.writeFile(
+            path.join(libDir, 'metadata.json'),
+            JSON.stringify({
+                module: 'github.com/google/uuid',
+                version: 'v1.6.0',
+                go_mod_require: 'github.com/google/uuid v1.6.0',
+                go_sum: 'github.com/google/uuid v1.6.0 h1:abc123=',
+            }),
+        );
+
+        await (compiler as any).generateGoMod(tempDir, ['uuid'], tempDir);
+
+        const goModContent = await fsp.readFile(path.join(tempDir, 'go.mod'), 'utf-8');
+        expect(goModContent).toContain('module example');
+        expect(goModContent).toContain('github.com/google/uuid v1.6.0');
+
+        const goSumContent = await fsp.readFile(path.join(tempDir, 'go.sum'), 'utf-8');
+        expect(goSumContent).toContain('github.com/google/uuid v1.6.0 h1:abc123=');
+    });
+
+    it('Appends to existing go.mod with require block', async () => {
+        const existingGoMod = `module mymodule
+
+go 1.20
+
+require (
+\texisting/module v1.0.0
+)
+`;
+        await fsp.writeFile(path.join(tempDir, 'go.mod'), existingGoMod);
+
+        const libDir = path.join(tempDir, 'uuid');
+        await fsp.mkdir(libDir, {recursive: true});
+        await fsp.writeFile(
+            path.join(libDir, 'metadata.json'),
+            JSON.stringify({
+                module: 'github.com/google/uuid',
+                version: 'v1.6.0',
+                go_mod_require: 'github.com/google/uuid v1.6.0',
+                go_sum: '',
+            }),
+        );
+
+        await (compiler as any).generateGoMod(tempDir, ['uuid'], tempDir);
+
+        const goModContent = await fsp.readFile(path.join(tempDir, 'go.mod'), 'utf-8');
+        expect(goModContent).toContain('module mymodule');
+        expect(goModContent).toContain('existing/module v1.0.0');
+        expect(goModContent).toContain('github.com/google/uuid v1.6.0');
+    });
+
+    it('Merges cache delta into GOCACHE', async () => {
+        const cachePath = path.join(tempDir, 'cache');
+        const cacheDeltaPath = path.join(tempDir, 'lib', 'cache_delta');
+
+        await fsp.mkdir(cachePath, {recursive: true});
+        await fsp.mkdir(cacheDeltaPath, {recursive: true});
+
+        // Create existing cache file
+        await fsp.writeFile(path.join(cachePath, 'existing.txt'), 'existing content');
+
+        // Create cache delta files
+        await fsp.writeFile(path.join(cacheDeltaPath, 'new_file.txt'), 'new content');
+
+        await (compiler as any).mergeGocache(cachePath, cacheDeltaPath);
+
+        // Both files should exist
+        expect(await utils.fileExists(path.join(cachePath, 'existing.txt'))).toBe(true);
+        expect(await utils.fileExists(path.join(cachePath, 'new_file.txt'))).toBe(true);
+
+        const newFileContent = await fsp.readFile(path.join(cachePath, 'new_file.txt'), 'utf-8');
+        expect(newFileContent).toBe('new content');
+    });
+
+    it('Sets up module sources in GOPATH', async () => {
+        const goPath = path.join(tempDir, 'gopath');
+        const libPath = path.join(tempDir, 'uuid');
+        const moduleSourcesPath = path.join(libPath, 'module_sources');
+
+        await fsp.mkdir(goPath, {recursive: true});
+        await fsp.mkdir(moduleSourcesPath, {recursive: true});
+
+        // Create module source files
+        const modulePath = path.join(moduleSourcesPath, 'github.com', 'google', 'uuid@v1.6.0');
+        await fsp.mkdir(modulePath, {recursive: true});
+        await fsp.writeFile(path.join(modulePath, 'uuid.go'), 'package uuid');
+
+        await (compiler as any).setupModuleSources(goPath, libPath);
+
+        const destPath = path.join(goPath, 'pkg', 'mod', 'github.com', 'google', 'uuid@v1.6.0', 'uuid.go');
+        expect(await utils.fileExists(destPath)).toBe(true);
+    });
+});