From eecaef4ef3494449a63ff8da8a73eb335cfdb187 Mon Sep 17 00:00:00 2001 From: Patrick Quist Date: Sun, 8 Feb 2026 01:18:43 +0100 Subject: [PATCH] Add library support for Go compiler (#8397) --- .claude/commands/resetprops.md | 9 ++ etc/config/go.amazon.properties | 20 +++- lib/buildenvsetup/_all.ts | 1 + lib/buildenvsetup/ceconan-go.ts | 99 ++++++++++++++++++ lib/compilers/golang.ts | 178 +++++++++++++++++++++++++++++++- test/golang-tests.ts | 171 +++++++++++++++++++++++++++++- 6 files changed, 471 insertions(+), 7 deletions(-) create mode 100644 .claude/commands/resetprops.md create mode 100644 lib/buildenvsetup/ceconan-go.ts diff --git a/.claude/commands/resetprops.md b/.claude/commands/resetprops.md new file mode 100644 index 000000000..a6b8af6c7 --- /dev/null +++ b/.claude/commands/resetprops.md @@ -0,0 +1,9 @@ +Reset all `.local.properties` files in `etc/config/` by overwriting them with the contents of their corresponding `.amazon.properties` files. + +Steps: +1. Find all `*.local.properties` files in `etc/config/` +2. For each file, determine the base name (e.g. `c++.local.properties` -> `c++`) +3. Check if a corresponding `*.amazon.properties` file exists (e.g. `c++.amazon.properties`) +4. If it exists, overwrite the `.local.properties` file with the contents of the `.amazon.properties` file +5. If no corresponding `.amazon.properties` file exists, skip it and note that it was skipped +6. Report a summary of which files were overwritten and which were skipped diff --git a/etc/config/go.amazon.properties b/etc/config/go.amazon.properties index 586918b96..45bcc4c52 100644 --- a/etc/config/go.amazon.properties +++ b/etc/config/go.amazon.properties @@ -107,6 +107,8 @@ group.gl.compilers=&x86gl:&armgl:&mipsgl:&ppcgl:&riscvgl:&s390xgl:&wasmgl group.gl.versionFlag=version group.gl.compilerType=golang group.gl.isSemVer=true +group.gl.buildenvsetup=ceconan-go +group.gl.buildenvsetup.host=https://conan.compiler-explorer.com ###### x86 GC ###### group.x86gl.compilers=&amd64gl:&386gl @@ -1589,7 +1591,23 @@ compiler.gppc64g9.semver=AT13.0 ################################# ################################# # Installed libs (See c++.amazon.properties for a scheme of libs group) -libs= + +libs=uuid:protobuf:errors + +libs.uuid.name=github.com/google/uuid +libs.uuid.url=https://github.com/google/uuid +libs.uuid.versions=v160 +libs.uuid.versions.v160.version=v1.6.0 + +libs.protobuf.name=google.golang.org/protobuf +libs.protobuf.url=https://github.com/protocolbuffers/protobuf-go +libs.protobuf.versions=v1360 +libs.protobuf.versions.v1360.version=v1.36.0 + +libs.errors.name=github.com/pkg/errors +libs.errors.url=https://github.com/pkg/errors +libs.errors.versions=v091 +libs.errors.versions.v091.version=v0.9.1 ################################# ################################# diff --git a/lib/buildenvsetup/_all.ts b/lib/buildenvsetup/_all.ts index 4190f0b38..fdc9495b5 100644 --- a/lib/buildenvsetup/_all.ts +++ b/lib/buildenvsetup/_all.ts @@ -25,4 +25,5 @@ export {BuildEnvSetupCeConanDirect} from './ceconan.js'; export {BuildEnvSetupCeConanCircleDirect} from './ceconan-circle.js'; export {BuildEnvSetupCeConanFortranDirect} from './ceconan-fortran.js'; +export {BuildEnvSetupCeConanGoDirect} from './ceconan-go.js'; export {BuildEnvSetupCeConanRustDirect} from './ceconan-rust.js'; diff --git a/lib/buildenvsetup/ceconan-go.ts b/lib/buildenvsetup/ceconan-go.ts new file mode 100644 index 000000000..ad32044a4 --- /dev/null +++ b/lib/buildenvsetup/ceconan-go.ts @@ -0,0 +1,99 @@ +// Copyright (c) 2025, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import _ from 'underscore'; + +import {CacheKey} from '../../types/compilation/compilation.interfaces.js'; +import {CompilerInfo} from '../../types/compiler.interfaces.js'; +import {CompilationEnvironment} from '../compilation-env.js'; +import {VersionInfo} from '../options-handler.js'; +import {ExecCompilerCachedFunc} from './base.js'; +import type {BuildEnvDownloadInfo} from './buildenv.interfaces.js'; +import {BuildEnvSetupCeConanDirect} from './ceconan.js'; + +export class BuildEnvSetupCeConanGoDirect extends BuildEnvSetupCeConanDirect { + static override get key() { + return 'ceconan-go'; + } + + constructor(compilerInfo: CompilerInfo, env: CompilationEnvironment) { + super(compilerInfo, env); + + this.onlyonstaticliblink = false; + this.extractAllToRoot = false; + } + + override async initialise(execCompilerCachedFunc: ExecCompilerCachedFunc) { + if (this.compilerArch) return; + this.compilerSupportsX86 = true; + } + + override getLibcxx(key: CacheKey) { + return ''; + } + + override getTarget(key: CacheKey) { + const goarch = (this.compiler.goarch || 'amd64').toString(); + return BuildEnvSetupCeConanGoDirect.goArchToConanArch(goarch); + } + + static goArchToConanArch(goarch: string): string { + switch (goarch) { + case 'amd64': + return 'x86_64'; + case '386': + return 'x86'; + case 'arm64': + return 'aarch64'; + default: + return goarch; + } + } + + override hasBinariesToLink(details: VersionInfo) { + return true; + } + + override shouldDownloadPackage(details: VersionInfo) { + return true; + } + + override async download( + key: CacheKey, + dirPath: string, + libraryDetails: Record, + ): Promise { + const modifiedLibraryDetails: Record = {}; + + _.each(libraryDetails, (details: VersionInfo, libId: string) => { + const modifiedDetails = {...details}; + if (!modifiedDetails.lookupname) { + modifiedDetails.lookupname = `go_${libId}`; + } + modifiedLibraryDetails[libId] = modifiedDetails; + }); + + return super.download(key, dirPath, modifiedLibraryDetails); + } +} diff --git a/lib/compilers/golang.ts b/lib/compilers/golang.ts index 569fcf826..9c2c875d1 100644 --- a/lib/compilers/golang.ts +++ b/lib/compilers/golang.ts @@ -27,16 +27,25 @@ import path from 'node:path'; import _ from 'underscore'; -import type {ExecutionOptionsWithEnv} from '../../types/compilation/compilation.interfaces.js'; +import type {CacheKey, ExecutionOptionsWithEnv} from '../../types/compilation/compilation.interfaces.js'; import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; import type {ResultLine} from '../../types/resultline/resultline.interfaces.js'; import {unwrap} from '../assert.js'; import {BaseCompiler} from '../base-compiler.js'; +import type {BuildEnvDownloadInfo} from '../buildenvsetup/buildenv.interfaces.js'; import {CompilationEnvironment} from '../compilation-env.js'; +import {logger} from '../logger.js'; import * as utils from '../utils.js'; import {GolangParser} from './argument-parsers.js'; +interface GoLibraryMetadata { + module: string; + version: string; + go_mod_require: string; + go_sum: string; +} + // Each arch has a list of jump instructions in // Go source src/cmd/asm/internal/arch. // x86 -> j, b @@ -57,6 +66,8 @@ type GoEnv = { export class GolangCompiler extends BaseCompiler { private readonly GOENV: GoEnv; + private hasLibraries = false; + private verboseBuild = false; static get key() { return 'golang'; @@ -107,6 +118,137 @@ export class GolangCompiler extends BaseCompiler { return undefined; } + override async setupBuildEnvironment( + key: CacheKey, + dirPath: string, + binary: boolean, + ): Promise { + this.hasLibraries = key.libraries && key.libraries.length > 0; + return super.setupBuildEnvironment(key, dirPath, binary); + } + + protected async findDownloadedLibraries(dirPath: string): Promise { + const libraries: string[] = []; + try { + const entries = await fs.readdir(dirPath, {withFileTypes: true}); + for (const entry of entries) { + if (entry.isDirectory()) { + const metadataPath = path.join(dirPath, entry.name, 'metadata.json'); + if (await utils.fileExists(metadataPath)) { + libraries.push(entry.name); + } + } + } + } catch { + // Directory doesn't exist or can't be read + } + return libraries; + } + + protected async readLibraryMetadata(dirPath: string, libId: string): Promise { + try { + const metadataPath = path.join(dirPath, libId, 'metadata.json'); + const content = await fs.readFile(metadataPath, 'utf-8'); + return JSON.parse(content) as GoLibraryMetadata; + } catch (e) { + logger.warn(`Failed to read metadata for Go library ${libId}: ${e}`); + return null; + } + } + + protected async mergeGocache(cachePath: string, libCacheDeltaPath: string): Promise { + if (!(await utils.dirExists(libCacheDeltaPath))) return; + await fs.cp(libCacheDeltaPath, cachePath, {recursive: true, force: false}); + } + + protected async setupModuleSources(goPath: string, libPath: string): Promise { + const moduleSourcesPath = path.join(libPath, 'module_sources'); + if (!(await utils.dirExists(moduleSourcesPath))) return; + + const pkgModPath = path.join(goPath, 'pkg', 'mod'); + await fs.mkdir(pkgModPath, {recursive: true}); + await fs.cp(moduleSourcesPath, pkgModPath, {recursive: true, force: false}); + } + + protected async generateGoMod(inputDir: string, libraries: string[], dirPath: string): Promise { + const goModPath = path.join(inputDir, 'go.mod'); + const goSumPath = path.join(inputDir, 'go.sum'); + + let goModContent = ''; + let goSumContent = ''; + + // Check if user provided their own go.mod + const existingGoMod = await utils.fileExists(goModPath); + if (existingGoMod) { + goModContent = await fs.readFile(goModPath, 'utf-8'); + } else { + goModContent = 'module example\n\ngo 1.21\n'; + } + + // Collect require statements and sum entries from all libraries + const requireStatements: string[] = []; + const sumEntries: string[] = []; + + for (const libId of libraries) { + const metadata = await this.readLibraryMetadata(dirPath, libId); + if (metadata) { + if (metadata.go_mod_require) { + requireStatements.push(metadata.go_mod_require); + } + if (metadata.go_sum) { + sumEntries.push(metadata.go_sum); + } + } + } + + // Append require statements to go.mod + if (requireStatements.length > 0) { + if (!goModContent.includes('require (')) { + goModContent += '\nrequire (\n'; + goModContent += requireStatements.map(r => `\t${r}`).join('\n'); + goModContent += '\n)\n'; + } else { + // Insert before the closing paren of require block + goModContent = goModContent.replace( + /require \(([^)]*)\)/, + (match, inner) => `require (${inner}\n${requireStatements.map(r => `\t${r}`).join('\n')}\n)`, + ); + } + } + + await fs.writeFile(goModPath, goModContent); + + // Write go.sum + if (sumEntries.length > 0) { + goSumContent = sumEntries.join('\n') + '\n'; + await fs.writeFile(goSumPath, goSumContent); + } + } + + protected async setupGoLibraries( + inputDir: string, + cachePath: string, + goPath: string, + dirPath: string, + ): Promise { + const libraries = await this.findDownloadedLibraries(dirPath); + if (libraries.length === 0) return; + + for (const libId of libraries) { + const libPath = path.join(dirPath, libId); + + // Merge cache_delta into GOCACHE + const cacheDeltaPath = path.join(libPath, 'cache_delta'); + await this.mergeGocache(cachePath, cacheDeltaPath); + + // Copy module_sources to GOPATH/pkg/mod + await this.setupModuleSources(goPath, libPath); + } + + // Generate go.mod and go.sum + await this.generateGoMod(inputDir, libraries, dirPath); + } + override async runCompiler( compiler: string, options: string[], @@ -119,24 +261,43 @@ export class GolangCompiler extends BaseCompiler { } const inputDir = path.dirname(inputFilename); + const dirPath = inputDir; const tempCachePath = path.join(inputDir, 'cache'); + const goPath = path.join(inputDir, 'gopath'); execOptions.env = { ...execOptions.env, GOCACHE: tempCachePath, + GOPATH: goPath, }; + this.verboseBuild = options.some(opt => opt === '-v' || opt === '-x'); + + // Force offline compilation when libraries are selected + if (this.hasLibraries) { + execOptions.env.GOPROXY = 'off'; + } + const sourceCachePath = await this.getSourceCachePath(); if (sourceCachePath) { try { await fs.mkdir(tempCachePath, {recursive: true}); - await fs.cp(sourceCachePath, tempCachePath, {recursive: true, force: false}); } catch { // Cache setup failed, continue without cache } } + // Set up Go libraries if any were downloaded + if (this.hasLibraries) { + try { + await fs.mkdir(goPath, {recursive: true}); + await this.setupGoLibraries(inputDir, tempCachePath, goPath, dirPath); + } catch (e) { + logger.warn(`Failed to set up Go libraries: ${e}`); + } + } + return super.runCompiler(compiler, options, inputFilename, execOptions, filters); } @@ -277,7 +438,9 @@ export class GolangCompiler extends BaseCompiler { } const logging = this.extractLogging(out); result.asm = this.convertNewGoL(out); - result.stderr = []; + result.stderr = this.verboseBuild + ? out.filter(obj => !obj.text.match(LINE_RE) && !obj.text.match(UNKNOWN_RE) && !obj.text.match(LOGGING_RE)) + : []; result.stdout = utils.parseOutput(logging, result.inputFilename); return Promise.all([result, [], []]); } @@ -286,6 +449,11 @@ export class GolangCompiler extends BaseCompiler { return []; } + override getIncludeArguments(libraries: object[]): string[] { + // Go uses the module system, not include flags + return []; + } + override optionsForFilter(filters: ParseFiltersAndOutputOptions, outputFilename: string, userOptions?: string[]) { // If we're dealing with an older version... if (this.compiler.id === '6g141') { @@ -293,10 +461,10 @@ export class GolangCompiler extends BaseCompiler { } if (filters.binary) { - return ['build', '-o', outputFilename, '-gcflags=' + unwrap(userOptions).join(' ')]; + return ['build', '-trimpath', '-o', outputFilename, '-gcflags=' + unwrap(userOptions).join(' ')]; } // Add userOptions to -gcflags to preserve previous behavior. - return ['build', '-o', outputFilename, '-gcflags=-S ' + unwrap(userOptions).join(' ')]; + return ['build', '-trimpath', '-o', outputFilename, '-gcflags=-S ' + unwrap(userOptions).join(' ')]; } override filterUserOptions(userOptions: string[]) { diff --git a/test/golang-tests.ts b/test/golang-tests.ts index 92bcdf21b..9b3c7dd9b 100644 --- a/test/golang-tests.ts +++ b/test/golang-tests.ts @@ -23,8 +23,11 @@ // POSSIBILITY OF SUCH DAMAGE. import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; -import {beforeAll, describe, expect, it} from 'vitest'; +import {afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest'; import {CompilationEnvironment} from '../lib/compilation-env.js'; import {GolangCompiler} from '../lib/compilers/golang.js'; @@ -123,3 +126,169 @@ describe('GO environment variables', () => { expect(execOptions.env.GOCACHE).toBeUndefined(); }); }); + +describe('GO library support', () => { + let tempDir: string; + let compiler: GolangCompiler; + + beforeAll(() => { + ce = makeCompilationEnvironment({languages}); + }); + + beforeEach(async () => { + tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'ce-go-test-')); + const compilerInfo = makeFakeCompilerInfo({ + exe: '/opt/compiler-explorer/go1.20/bin/go', + lang: languages.go.id, + }); + compiler = new GolangCompiler(compilerInfo, ce); + }); + + afterEach(async () => { + await fsp.rm(tempDir, {recursive: true, force: true}); + vi.restoreAllMocks(); + }); + + it('Returns empty array for getIncludeArguments', () => { + const result = compiler.getIncludeArguments([{paths: ['/some/path']}]); + expect(result).toEqual([]); + }); + + it('Finds downloaded libraries with metadata.json', async () => { + // Create mock library directories + const libDir = path.join(tempDir, 'uuid'); + await fsp.mkdir(libDir, {recursive: true}); + await fsp.writeFile( + path.join(libDir, 'metadata.json'), + JSON.stringify({ + module: 'github.com/google/uuid', + version: 'v1.6.0', + go_mod_require: 'github.com/google/uuid v1.6.0', + go_sum: 'github.com/google/uuid v1.6.0 h1:abc123=', + }), + ); + + // Also create a non-library directory + const nonLibDir = path.join(tempDir, 'cache'); + await fsp.mkdir(nonLibDir, {recursive: true}); + + const libraries = await (compiler as any).findDownloadedLibraries(tempDir); + expect(libraries).toEqual(['uuid']); + }); + + it('Reads library metadata correctly', async () => { + const libDir = path.join(tempDir, 'uuid'); + await fsp.mkdir(libDir, {recursive: true}); + const metadata = { + module: 'github.com/google/uuid', + version: 'v1.6.0', + go_mod_require: 'github.com/google/uuid v1.6.0', + go_sum: 'github.com/google/uuid v1.6.0 h1:abc123=', + }; + await fsp.writeFile(path.join(libDir, 'metadata.json'), JSON.stringify(metadata)); + + const result = await (compiler as any).readLibraryMetadata(tempDir, 'uuid'); + expect(result).toEqual(metadata); + }); + + it('Returns null for missing metadata', async () => { + const result = await (compiler as any).readLibraryMetadata(tempDir, 'nonexistent'); + expect(result).toBeNull(); + }); + + it('Generates go.mod with require statements', async () => { + const libDir = path.join(tempDir, 'uuid'); + await fsp.mkdir(libDir, {recursive: true}); + await fsp.writeFile( + path.join(libDir, 'metadata.json'), + JSON.stringify({ + module: 'github.com/google/uuid', + version: 'v1.6.0', + go_mod_require: 'github.com/google/uuid v1.6.0', + go_sum: 'github.com/google/uuid v1.6.0 h1:abc123=', + }), + ); + + await (compiler as any).generateGoMod(tempDir, ['uuid'], tempDir); + + const goModContent = await fsp.readFile(path.join(tempDir, 'go.mod'), 'utf-8'); + expect(goModContent).toContain('module example'); + expect(goModContent).toContain('github.com/google/uuid v1.6.0'); + + const goSumContent = await fsp.readFile(path.join(tempDir, 'go.sum'), 'utf-8'); + expect(goSumContent).toContain('github.com/google/uuid v1.6.0 h1:abc123='); + }); + + it('Appends to existing go.mod with require block', async () => { + const existingGoMod = `module mymodule + +go 1.20 + +require ( +\texisting/module v1.0.0 +) +`; + await fsp.writeFile(path.join(tempDir, 'go.mod'), existingGoMod); + + const libDir = path.join(tempDir, 'uuid'); + await fsp.mkdir(libDir, {recursive: true}); + await fsp.writeFile( + path.join(libDir, 'metadata.json'), + JSON.stringify({ + module: 'github.com/google/uuid', + version: 'v1.6.0', + go_mod_require: 'github.com/google/uuid v1.6.0', + go_sum: '', + }), + ); + + await (compiler as any).generateGoMod(tempDir, ['uuid'], tempDir); + + const goModContent = await fsp.readFile(path.join(tempDir, 'go.mod'), 'utf-8'); + expect(goModContent).toContain('module mymodule'); + expect(goModContent).toContain('existing/module v1.0.0'); + expect(goModContent).toContain('github.com/google/uuid v1.6.0'); + }); + + it('Merges cache delta into GOCACHE', async () => { + const cachePath = path.join(tempDir, 'cache'); + const cacheDeltaPath = path.join(tempDir, 'lib', 'cache_delta'); + + await fsp.mkdir(cachePath, {recursive: true}); + await fsp.mkdir(cacheDeltaPath, {recursive: true}); + + // Create existing cache file + await fsp.writeFile(path.join(cachePath, 'existing.txt'), 'existing content'); + + // Create cache delta files + await fsp.writeFile(path.join(cacheDeltaPath, 'new_file.txt'), 'new content'); + + await (compiler as any).mergeGocache(cachePath, cacheDeltaPath); + + // Both files should exist + expect(await utils.fileExists(path.join(cachePath, 'existing.txt'))).toBe(true); + expect(await utils.fileExists(path.join(cachePath, 'new_file.txt'))).toBe(true); + + const newFileContent = await fsp.readFile(path.join(cachePath, 'new_file.txt'), 'utf-8'); + expect(newFileContent).toBe('new content'); + }); + + it('Sets up module sources in GOPATH', async () => { + const goPath = path.join(tempDir, 'gopath'); + const libPath = path.join(tempDir, 'uuid'); + const moduleSourcesPath = path.join(libPath, 'module_sources'); + + await fsp.mkdir(goPath, {recursive: true}); + await fsp.mkdir(moduleSourcesPath, {recursive: true}); + + // Create module source files + const modulePath = path.join(moduleSourcesPath, 'github.com', 'google', 'uuid@v1.6.0'); + await fsp.mkdir(modulePath, {recursive: true}); + await fsp.writeFile(path.join(modulePath, 'uuid.go'), 'package uuid'); + + await (compiler as any).setupModuleSources(goPath, libPath); + + const destPath = path.join(goPath, 'pkg', 'mod', 'github.com', 'google', 'uuid@v1.6.0', 'uuid.go'); + expect(await utils.fileExists(destPath)).toBe(true); + }); +});