From 02603a6371586770afa71b6968f80515a12f4017 Mon Sep 17 00:00:00 2001 From: Spencer Fricke <115671160+spencer-lunarg@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:24:50 -0400 Subject: [PATCH] Add SPIR-V language with SPIRV-Tools compilers (#6947) for https://github.com/compiler-explorer/compiler-explorer/issues/2331 Hey, so I am currently trying to improve the SPIR-V ecosystem, in-charge of the [SPIR-V Guide](https://github.com/KhronosGroup/SPIRV-Guide), and part of the SPIR-V Working Group. We talked at our last Face-2-Face gathering about getting more support for GPU tooling in something like Compiler Explorer because you have all made this such an amazing tool! I have enjoyed working in this codebase and happy to help keep improving the SPIR-V as people find issues or want enhancements (I was already about to spend time rebuilding Compiler Explorer, so happy to donate my time to a single tool's effort) (For those who haven't spent last few years staring at SPIR-V) - SPIR-V is a an IR (very much like LLVM IR) but for GPUs - SPIR-V can be used for Graphics and Compute - Compute: - Can be tied to things like clang, llvm, mlir, OpenCL, etc - One could take `spirv` as a language and compile it to something like x86 - (Future PRs for this realm of things) - Graphics: - Much harder to actually "compile" to any ISA because other graphics pipeline information is missing that will effect the final ISA - Microsoft has even now planned to move to [SPIR-V for DirectX soon](https://devblogs.microsoft.com/directx/directx-adopting-spir-v/) - Both: - There are many tools that run on SPIR-V regardless (hence why they share the same IR) - Being able to do tooling in compiler explorer would be AMAZING This PR adds `SPIR-V` as a language and a new `SPIRV-Tools` compiler as an initial compiler for it the language (since we already have the SPIRV-Tools repo being pulled in on the infra side!) I personally use `spirv-opt` and `spirv-val` the most and have been testing this locally for a few days and feel it is ready to go. I plan to slowly add more tools (ex. `spirv-fuzz`, `spirv-cross`, `spirv-reflect`, etc) Cheers! (some screenshots of course) ![image](https://github.com/user-attachments/assets/419feab0-d030-4578-b32a-ef1cf95701da) ![image](https://github.com/user-attachments/assets/a00fa60f-7f05-4522-a9fc-1c43ecee1c42) --- .github/labeler.yml | 8 ++ etc/config/spirv.amazon.properties | 21 ++++ etc/config/spirv.defaults.properties | 21 ++++ examples/spirv/default.spvasm | 36 ++++++ lib/compilers/_all.ts | 1 + lib/compilers/spirv-tools.ts | 176 +++++++++++++++++++++++++++ lib/compilers/spirv.ts | 3 +- lib/languages.ts | 11 ++ types/languages.interfaces.ts | 1 + views/resources/logos/spirv.svg | 42 +++++++ 10 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 etc/config/spirv.amazon.properties create mode 100644 etc/config/spirv.defaults.properties create mode 100644 examples/spirv/default.spvasm create mode 100644 lib/compilers/spirv-tools.ts create mode 100644 views/resources/logos/spirv.svg diff --git a/.github/labeler.yml b/.github/labeler.yml index d27cfc4f5..7657d4aea 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -340,6 +340,14 @@ - 'etc/config/spice.*.properties' - 'static/modes/spice-mode.ts' +'lang-spirv': + - changed-files: + - any-glob-to-any-file: + - 'lib/compilers/spirv.ts' + - 'lib/compilers/spirv-tools.ts' + - 'etc/config/spirv.*.properties' + - 'static/modes/spirv-mode.ts' + 'lang-swift': - changed-files: - any-glob-to-any-file: diff --git a/etc/config/spirv.amazon.properties b/etc/config/spirv.amazon.properties new file mode 100644 index 000000000..f9275f069 --- /dev/null +++ b/etc/config/spirv.amazon.properties @@ -0,0 +1,21 @@ +compilers=&spirv-opt:&spirv-val +defaultCompiler=spirv-opt + +supportsBinary=false +supportsExecute=false +supportsAsmDocs=false + +group.spirv-opt.groupName=Optimizer +group.spirv-opt.baseName=SPIRV-Tools Optimizer +group.spirv-opt.compilers=spirv-opt +group.spirv-opt.compilerType=spirv-tools + +group.spirv-val.groupName=Validator +group.spirv-val.baseName=SPIRV-Tools Validator +group.spirv-val.compilers=spirv-val +group.spirv-val.compilerType=spirv-tools + +assemblerPath=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-as +disassemblerPath=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-dis +compiler.spirv-opt.exe=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-opt +compiler.spirv-val.exe=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-val diff --git a/etc/config/spirv.defaults.properties b/etc/config/spirv.defaults.properties new file mode 100644 index 000000000..f9275f069 --- /dev/null +++ b/etc/config/spirv.defaults.properties @@ -0,0 +1,21 @@ +compilers=&spirv-opt:&spirv-val +defaultCompiler=spirv-opt + +supportsBinary=false +supportsExecute=false +supportsAsmDocs=false + +group.spirv-opt.groupName=Optimizer +group.spirv-opt.baseName=SPIRV-Tools Optimizer +group.spirv-opt.compilers=spirv-opt +group.spirv-opt.compilerType=spirv-tools + +group.spirv-val.groupName=Validator +group.spirv-val.baseName=SPIRV-Tools Validator +group.spirv-val.compilers=spirv-val +group.spirv-val.compilerType=spirv-tools + +assemblerPath=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-as +disassemblerPath=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-dis +compiler.spirv-opt.exe=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-opt +compiler.spirv-val.exe=/opt/compiler-explorer/SPIRV-Tools-master/build/tools/spirv-val diff --git a/examples/spirv/default.spvasm b/examples/spirv/default.spvasm new file mode 100644 index 000000000..6ad18dd2d --- /dev/null +++ b/examples/spirv/default.spvasm @@ -0,0 +1,36 @@ +; SPIR-V +; Version: 1.0 +; Generator: Khronos Glslang Reference Front End; 7 +; Bound: 19 +; Schema: 0 + OpCapability Shader + %1 = OpExtInstImport "GLSL.std.450" + OpMemoryModel Logical GLSL450 + OpEntryPoint Fragment %main "main" %outFragColor %inColor + OpExecutionMode %main OriginUpperLeft + OpSource GLSL 450 + OpName %main "main" + OpName %outFragColor "outFragColor" + OpName %inColor "inColor" + OpDecorate %outFragColor Location 0 + OpDecorate %inColor Location 0 + %void = OpTypeVoid + %3 = OpTypeFunction %void + %float = OpTypeFloat 32 + %v4float = OpTypeVector %float 4 +%_ptr_Output_v4float = OpTypePointer Output %v4float +%outFragColor = OpVariable %_ptr_Output_v4float Output + %v3float = OpTypeVector %float 3 +%_ptr_Input_v3float = OpTypePointer Input %v3float + %inColor = OpVariable %_ptr_Input_v3float Input + %float_1 = OpConstant %float 1 + %main = OpFunction %void None %3 + %5 = OpLabel + %13 = OpLoad %v3float %inColor + %15 = OpCompositeExtract %float %13 0 + %16 = OpCompositeExtract %float %13 1 + %17 = OpCompositeExtract %float %13 2 + %18 = OpCompositeConstruct %v4float %15 %16 %17 %float_1 + OpStore %outFragColor %18 + OpReturn + OpFunctionEnd diff --git a/lib/compilers/_all.ts b/lib/compilers/_all.ts index c8e61adf9..7d181e882 100644 --- a/lib/compilers/_all.ts +++ b/lib/compilers/_all.ts @@ -126,6 +126,7 @@ export {SolidityCompiler} from './solidity.js'; export {SolidityZKsyncCompiler} from './solidity-zksync.js'; export {SpiceCompiler} from './spice.js'; export {SPIRVCompiler} from './spirv.js'; +export {SPIRVToolsCompiler} from './spirv-tools.js'; export {SwiftCompiler} from './swift.js'; export {TableGenCompiler} from './tablegen.js'; export {TenDRACompiler} from './tendra.js'; diff --git a/lib/compilers/spirv-tools.ts b/lib/compilers/spirv-tools.ts new file mode 100644 index 000000000..54a5b87e2 --- /dev/null +++ b/lib/compilers/spirv-tools.ts @@ -0,0 +1,176 @@ +// Copyright (c) 2024, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import path from 'path'; + +import type {ExecutionOptionsWithEnv} from '../../types/compilation/compilation.interfaces.js'; +import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; +import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; +import {BaseCompiler} from '../base-compiler.js'; +import {CompilationEnvironment} from '../compilation-env.js'; +import {logger} from '../logger.js'; +import {SPIRVAsmParser} from '../parsers/asm-parser-spirv.js'; +import * as utils from '../utils.js'; + +// If you want to output SPIR-V, most likely you want SPIRVAsmParser +// +// SPIR-V is an IR that targets heterogeneous (like a GPU) and has a "compute" and "graphics" mode. +// When used for graphics, it inself is not enough info to "compile" down to the GPU's assembly because normally there +// is other graphics // pipeline information that is required (ex. is the depth test enable/disabled) +// +// SPIR-V has a lot of tooling around it to optimize, validate, fuzz, etc. This compiler is only used for tooling. +export class SPIRVToolsCompiler extends BaseCompiler { + protected assemblerPath: string; + protected disassemblerPath: string; + + static get key() { + return 'spirv-tools'; + } + + constructor(compilerInfo: PreliminaryCompilerInfo, env: CompilationEnvironment) { + super(compilerInfo, env); + + this.asm = new SPIRVAsmParser(this.compilerProps); + + // spirv-as + this.assemblerPath = this.compilerProps('assemblerPath'); + // spirv-dis + this.disassemblerPath = this.compilerProps('disassemblerPath'); + } + + override optionsForFilter(filters: ParseFiltersAndOutputOptions, outputFilename: string) { + const sourceDir = path.dirname(outputFilename); + const spvBinFilename = this.getPrimaryOutputFilename(sourceDir, this.outputFilebase); + return ['-o', spvBinFilename]; + } + + getPrimaryOutputFilename(dirPath: string, outputFilebase: string) { + return path.join(dirPath, `${outputFilebase}.spv`); + } + + override getOutputFilename(dirPath: string, outputFilebase: string) { + return path.join(dirPath, `${outputFilebase}.spvasm`); + } + + // Some tools (ex. spirv-opt) needs this as a single argument, so merge with "=" + mergeSpirvTargetEnv(options: string[]) { + const index = options.indexOf('--target-env'); + if (index !== -1) { + options.splice(index, 2, `--target-env=${options[index + 1]}`); + } + return options; + } + + // Some tools (ex. spirv-val) needs this as a two argument, so unmerge with "=" + unmergeSpirvTargetEnv(options: string[]) { + for (let i = 0; i < options.length; i++) { + if (options[i].indexOf('--target-env=') === 0) { + const parts = options[i].split('='); + options.splice(i, 1, parts[0], parts[1]); + break; + } + } + return options; + } + + getSpirvTargetEnv(options: string[]) { + const index = options.indexOf('--target-env'); + if (index !== -1) { + return [options[index], options[index + 1]]; + } + + for (const i in options) { + if (options[i].indexOf('--target-env=') === 0) { + return [options[i]]; + } + } + + return []; // no target found, use tool's default + } + + // Most flows follow the same flow: + // 1. Assemble from spirv disassembly to a spirv binary + // 2. Run the tool (which will dump out a binary) + // a. Most tools let you set the input and out file to the same binary file + // 3. Disassemble back to disassembly + override async runCompiler( + compiler: string, + options: string[], + inputFilename: string, + execOptions: ExecutionOptionsWithEnv, + ) { + const sourceDir = path.dirname(inputFilename); + const spvBinFilename = this.getPrimaryOutputFilename(sourceDir, this.outputFilebase); + + if (!execOptions) { + execOptions = this.getDefaultExecOptions(); + } + execOptions.customCwd = sourceDir; + + let assemblerFlags = [inputFilename, '-o', spvBinFilename]; + assemblerFlags = assemblerFlags.concat(this.getSpirvTargetEnv(options)); + // Will fail if input SPIR-V is so bad assembler can't understand, so should let user know + let spvasmOutput = await this.exec(this.assemblerPath, assemblerFlags, execOptions); + let result = this.transformToCompilationResult(spvasmOutput, inputFilename); + if (spvasmOutput.code !== 0 || !(await utils.fileExists(spvBinFilename))) { + return result; + } + + // needs to update options depending on the tool + const isValidator = compiler.endsWith('spirv-val'); + if (isValidator) { + // there is no output file, so remove what we added in optionsForFilter + options = options.splice(2); + options = this.unmergeSpirvTargetEnv(options); + } else { + options = this.mergeSpirvTargetEnv(options); + } + + // have tools input a binary and output it to same binary temp file + for (const i in options) { + if (options[i] === inputFilename) { + options[i] = spvBinFilename; + break; + } + } + + const spvBin = await this.exec(compiler, options, execOptions); + result = this.transformToCompilationResult(spvBin, inputFilename); + if (spvBin.code !== 0 || !(await utils.fileExists(spvBinFilename)) || isValidator) { + return result; + } + + const spvasmFilename = path.join(sourceDir, this.outputFilebase + '.spvasm'); + const disassemblerFlags = [spvBinFilename, '-o', spvasmFilename]; + + // Will likely never fail + spvasmOutput = await this.exec(this.disassemblerPath, disassemblerFlags, execOptions); + if (spvasmOutput.code !== 0) { + logger.error('spirv-dis failed to disassemble binary', spvasmOutput); + } + + result = this.transformToCompilationResult(spvasmOutput, spvBinFilename); + return result; + } +} diff --git a/lib/compilers/spirv.ts b/lib/compilers/spirv.ts index a75b4dcff..e32e0d3f9 100644 --- a/lib/compilers/spirv.ts +++ b/lib/compilers/spirv.ts @@ -1,4 +1,4 @@ -// Copyright (c) 2018, 2021, Compiler Explorer Authors, Arm Ltd +// Copyright (c) 2018, 2021, 2024 Compiler Explorer Authors, Arm Ltd // All rights reserved. // // Redistribution and use in source and binary forms, with or without @@ -38,6 +38,7 @@ import {logger} from '../logger.js'; import {SPIRVAsmParser} from '../parsers/asm-parser-spirv.js'; import * as utils from '../utils.js'; +// If you want to output SPIR-V, most likely you want SPIRVAsmParser export class SPIRVCompiler extends BaseCompiler { protected translatorPath: string; protected disassemblerPath: string; diff --git a/lib/languages.ts b/lib/languages.ts index c91794bbe..97be83619 100644 --- a/lib/languages.ts +++ b/lib/languages.ts @@ -726,6 +726,17 @@ const definitions: Record = { previewFilter: null, monacoDisassembly: null, }, + spirv: { + name: 'SPIR-V', + monaco: 'spirv', + extensions: ['.spvasm'], + alias: [], + logoUrl: 'spirv.svg', + logoUrlDark: null, + formatter: null, + previewFilter: null, + monacoDisassembly: null, + }, swift: { name: 'Swift', monaco: 'swift', diff --git a/types/languages.interfaces.ts b/types/languages.interfaces.ts index 7b602babb..a61631fc8 100644 --- a/types/languages.interfaces.ts +++ b/types/languages.interfaces.ts @@ -85,6 +85,7 @@ export type LanguageKey = | 'scala' | 'solidity' | 'spice' + | 'spirv' | 'swift' | 'tablegen' | 'toit' diff --git a/views/resources/logos/spirv.svg b/views/resources/logos/spirv.svg new file mode 100644 index 000000000..659ba381d --- /dev/null +++ b/views/resources/logos/spirv.svg @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + +