diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cb9c983ed..687e63cdb 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -174,4 +174,5 @@ From oldest to newest contributor, we would like to thank: - [Sean Garwood](https://github.com/sean-garwood) - [Victor Vianna](https://github.com/victorvianna) - [Piers Wombwell](https://github.com/pwombwell) +- [Sirui Mu](https://github.com/Lancern) - [Connor Simms](https://github.com/connorsimms) diff --git a/etc/scripts/ast_dump.py b/etc/scripts/ast_dump.py new file mode 100644 index 000000000..b82902cfb --- /dev/null +++ b/etc/scripts/ast_dump.py @@ -0,0 +1,45 @@ +"""Dumps the AST of a Python source file.""" + +import argparse +import ast +import sys + + +def main(): + parser = argparse.ArgumentParser( + description="Parse and dump the AST of an Python source file" + ) + parser.add_argument( + "input", + help="Path to input Python source code file", + ) + args = parser.parse_args() + + try: + with open(args.input, "r", encoding="utf-8") as f: + source = f.read() + except IOError as err: + # Log error and output empty array so CE handles it gracefully + print("Error reading file: %s" % err) + sys.exit(1) + + try: + tree = compile( + source, + filename="", + mode="exec", + flags=ast.PyCF_ONLY_AST, + dont_inherit=True, + optimize=0, + ) + except SyntaxError as err: + # Output the syntax error as a single entry so CE can display it + print("SyntaxError: %s" % err) + sys.exit(1) + + # The "ident" parameter is introduced in Python 3.9 + print(ast.dump(tree, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/lib/compilers/python.ts b/lib/compilers/python.ts index 0c7980366..733b7a69d 100644 --- a/lib/compilers/python.ts +++ b/lib/compilers/python.ts @@ -24,15 +24,20 @@ // POSSIBILITY OF SUCH DAMAGE. import type {AsmResultSource, ParsedAsmResultLine} from '../../types/asmresult/asmresult.interfaces.js'; +import type {CompilationResult} from '../../types/compilation/compilation.interfaces.js'; import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; +import type {ResultLine} from '../../types/resultline/resultline.interfaces.js'; import {BaseCompiler} from '../base-compiler.js'; import {CompilationEnvironment} from '../compilation-env.js'; +import {PythonAstParser} from '../python-ast.js'; import {resolvePathFromAppRoot} from '../utils.js'; import {BaseParser} from './argument-parsers.js'; export class PythonCompiler extends BaseCompiler { private readonly disasmScriptPath: string; + private readonly astScriptPath: string; + private readonly pythonAstParser: PythonAstParser; static get key() { return 'python'; @@ -45,6 +50,9 @@ export class PythonCompiler extends BaseCompiler { this.disasmScriptPath = this.compilerProps('disasmScript') || resolvePathFromAppRoot('etc', 'scripts', 'disasms', 'dis_all.py'); + this.astScriptPath = + this.compilerProps('astScript') || resolvePathFromAppRoot('etc', 'scripts', 'ast_dump.py'); + this.pythonAstParser = new PythonAstParser(this.compilerProps); } override async processAsm(result) { @@ -80,6 +88,35 @@ export class PythonCompiler extends BaseCompiler { return ['-I', this.disasmScriptPath, '--outputfile', outputFilename, '--inputfile']; } + override couldSupportASTDump(version: string) { + // Python's ast module is available since Python 2.6. However, the + // formatted dump of Python AST via `ast.dump()` is available since + // Python 3.9. Therefore, we return true for all versions >= 3.9. + + const versionRegex = /Python (\d+)\.(\d+)\.(\d+)$/; + const versionMatch = versionRegex.exec(version); + + if (versionMatch === null) { + return false; + } + + const major = Number.parseInt(versionMatch[1]); + const minor = Number.parseInt(versionMatch[2]); + return major > 3 || (major === 3 && minor >= 9); + } + + override async generateAST(inputFilename: string, options: string[]): Promise { + const astOptions = ['-I', this.astScriptPath, this.filename(inputFilename)]; + const execOptions = this.getDefaultExecOptions(); + const result: CompilationResult = await this.runCompiler( + this.compiler.exe, + astOptions, + this.filename(inputFilename), + execOptions, + ); + return this.pythonAstParser.processAst(result); + } + override getArgumentParserClass() { return BaseParser; } diff --git a/lib/python-ast.ts b/lib/python-ast.ts new file mode 100644 index 000000000..909e139bf --- /dev/null +++ b/lib/python-ast.ts @@ -0,0 +1,43 @@ +// Copyright (c) 2026, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import type {CompilationResult} from '../types/compilation/compilation.interfaces.js'; +import type {ResultLine} from '../types/resultline/resultline.interfaces.js'; +import type {PropertyGetter} from './properties.interfaces.js'; + +export class PythonAstParser { + private readonly maxAstLines: number; + + constructor(compilerProps: PropertyGetter) { + this.maxAstLines = 500; + if (compilerProps) { + this.maxAstLines = compilerProps('maxLinesOfAst', this.maxAstLines); + } + } + + processAst(result: CompilationResult): ResultLine[] { + const sliceEnd = Math.min(this.maxAstLines, result.stdout.length); + return result.stdout.slice(0, sliceEnd); + } +} diff --git a/test/ast/python-long.ast b/test/ast/python-long.ast new file mode 100644 index 000000000..2b8555556 --- /dev/null +++ b/test/ast/python-long.ast @@ -0,0 +1,160 @@ +Module( + body=[ + Expr( + value=Constant(value='Dumps the AST of a Python source file as structured JSON for Compiler Explorer.')), + Import( + names=[ + alias(name='argparse')]), + Import( + names=[ + alias(name='ast')]), + Import( + names=[ + alias(name='sys')]), + FunctionDef( + name='main', + args=arguments(), + body=[ + Assign( + targets=[ + Name(id='parser', ctx=Store())], + value=Call( + func=Attribute( + value=Name(id='argparse', ctx=Load()), + attr='ArgumentParser', + ctx=Load()), + keywords=[ + keyword( + arg='description', + value=Constant(value='Parse and dump the AST of an Python source file'))])), + Expr( + value=Call( + func=Attribute( + value=Name(id='parser', ctx=Load()), + attr='add_argument', + ctx=Load()), + args=[ + Constant(value='input')], + keywords=[ + keyword( + arg='help', + value=Constant(value='Path to input Python source code file'))])), + Assign( + targets=[ + Name(id='args', ctx=Store())], + value=Call( + func=Attribute( + value=Name(id='parser', ctx=Load()), + attr='parse_args', + ctx=Load()))), + Try( + body=[ + With( + items=[ + withitem( + context_expr=Call( + func=Name(id='open', ctx=Load()), + args=[ + Attribute( + value=Name(id='args', ctx=Load()), + attr='input', + ctx=Load()), + Constant(value='r')], + keywords=[ + keyword( + arg='encoding', + value=Constant(value='utf-8'))]), + optional_vars=Name(id='f', ctx=Store()))], + body=[ + Assign( + targets=[ + Name(id='source', ctx=Store())], + value=Call( + func=Attribute( + value=Name(id='f', ctx=Load()), + attr='read', + ctx=Load())))])], + handlers=[ + ExceptHandler( + type=Name(id='IOError', ctx=Load()), + name='err', + body=[ + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + BinOp( + left=Constant(value='Error reading file: %s'), + op=Mod(), + right=Name(id='err', ctx=Load()))])), + Expr( + value=Call( + func=Attribute( + value=Name(id='sys', ctx=Load()), + attr='exit', + ctx=Load()), + args=[ + Constant(value=1)]))])]), + Try( + body=[ + Assign( + targets=[ + Name(id='tree', ctx=Store())], + value=Call( + func=Attribute( + value=Name(id='ast', ctx=Load()), + attr='parse', + ctx=Load()), + args=[ + Name(id='source', ctx=Load())], + keywords=[ + keyword( + arg='optimize', + value=Constant(value=0))]))], + handlers=[ + ExceptHandler( + type=Name(id='SyntaxError', ctx=Load()), + name='err', + body=[ + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + BinOp( + left=Constant(value='SyntaxError: %s'), + op=Mod(), + right=Name(id='err', ctx=Load()))])), + Expr( + value=Call( + func=Attribute( + value=Name(id='sys', ctx=Load()), + attr='exit', + ctx=Load()), + args=[ + Constant(value=1)]))])]), + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + Call( + func=Attribute( + value=Name(id='ast', ctx=Load()), + attr='dump', + ctx=Load()), + args=[ + Name(id='tree', ctx=Load())], + keywords=[ + keyword( + arg='indent', + value=Constant(value=2))])]))]), + If( + test=Compare( + left=Name(id='__name__', ctx=Load()), + ops=[ + Eq()], + comparators=[ + Constant(value='__main__')]), + body=[ + Expr( + value=Call( + func=Name(id='main', ctx=Load())))])]) diff --git a/test/ast/python-short.ast b/test/ast/python-short.ast new file mode 100644 index 000000000..372ce76eb --- /dev/null +++ b/test/ast/python-short.ast @@ -0,0 +1,13 @@ +Module( + body=[ + FunctionDef( + name='square', + args=arguments( + args=[ + arg(arg='num')]), + body=[ + Return( + value=BinOp( + left=Name(id='num', ctx=Load()), + op=Mult(), + right=Name(id='num', ctx=Load())))])]) \ No newline at end of file diff --git a/test/python-ast-parser-tests.ts b/test/python-ast-parser-tests.ts new file mode 100644 index 000000000..15a8398dc --- /dev/null +++ b/test/python-ast-parser-tests.ts @@ -0,0 +1,71 @@ +// Copyright (c) 2026, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import fs from 'node:fs'; + +import {beforeAll, describe, expect, it} from 'vitest'; + +import * as properties from '../lib/properties.js'; +import {PythonAstParser} from '../lib/python-ast.js'; +import * as utils from '../lib/utils.js'; + +const languages = { + python: {id: 'python'}, +}; + +function mockAstOutput(astLines: string[]) { + return {stdout: astLines.map(l => ({text: l}))}; +} + +describe('python-ast', () => { + let compilerProps; + let astParser: any; + + let shortAstDump: string[]; + let longAstDump: string[]; + + beforeAll(() => { + const fakeProps = new properties.CompilerProps( + languages, + properties.fakeProps({ + maxLinesOfAst: 100, + }), + ); + compilerProps = (fakeProps.get as any).bind(fakeProps, 'python'); + astParser = new PythonAstParser(compilerProps); + + shortAstDump = utils.splitLines(fs.readFileSync('test/ast/python-short.ast').toString()); + longAstDump = utils.splitLines(fs.readFileSync('test/ast/python-long.ast').toString()); + }); + + it('keeps original lines on short ast output', () => { + const processedLines = astParser.processAst(mockAstOutput(shortAstDump)); + expect(processedLines.length).toBe(shortAstDump.length); + }); + + it('truncates long ast output', () => { + const processedLines = astParser.processAst(mockAstOutput(longAstDump)); + expect(processedLines.length).toBe(100); + }); +});