mirrors/compiler-explorer
2
0
Fork 0
mirror of https://github.com/compiler-explorer/compiler-explorer.git synced 2026-07-16 17:26:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

Enable AST output for Python (#8858)

Browse Source 
This patch adds support for viewing the abstract syntax tree (AST) of
Python source code in Compiler Explorer, matching the existing AST
viewer feature for C/C++ (Clang AST).

- This patch adds a helper script `ast_dump.py` that parses and dumps
the AST for a Python source code file. `python -m ast` should also work,
but it's not available in old versions of Python.
- This patch adds an AST parser for Python which mimics the structure of
the C/C++ AST parser. It also updates the Python compiler to enable AST
output.
- All existing and new tests pass.

Assisted-by: GitHub Copilot / DeepSeek v4 Flash
Assisted-by: GitHub Copilot / DeepSeek v4 Pro

<!-- THIS COMMENT IS INVISIBLE IN THE FINAL PR, BUT FEEL FREE TO REMOVE
IT
Thanks for taking the time to improve CE. We really appreciate it.
Before opening the PR, please make sure that the tests & linter pass
their checks,
  by running `make check`.
In the best case scenario, you are also adding tests to back up your
changes,
  but don't sweat it if you don't. We can discuss them at a later date.
Feel free to append your name to the CONTRIBUTORS.md file
Thanks again, we really appreciate this!
-->

Co-authored-by: Matt Godbolt <matt@godbolt.org>
This commit is contained in:
Sirui Mu
2026-07-14 15:46:10 +08:00
committed by GitHub
parent 2cc60cee5f
commit 7a6af91bc9
7 changed files with 370 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CONTRIBUTORS.md
View File

@@ -174,4 +174,5 @@ From oldest to newest contributor, we would like to thank:
- [Sean Garwood](https://github.com/sean-garwood)
- [Victor Vianna](https://github.com/victorvianna)
- [Piers Wombwell](https://github.com/pwombwell)
- [Sirui Mu](https://github.com/Lancern)
- [Connor Simms](https://github.com/connorsimms)

45
etc/scripts/ast_dump.py Normal file
View File

@@ -0,0 +1,45 @@
"""Dumps the AST of a Python source file."""
import argparse
import ast
import sys
def main():
parser = argparse.ArgumentParser(
description="Parse and dump the AST of an Python source file"
)
parser.add_argument(
"input",
help="Path to input Python source code file",
)
args = parser.parse_args()
try:
with open(args.input, "r", encoding="utf-8") as f:
source = f.read()
except IOError as err:
# Log error and output empty array so CE handles it gracefully
print("Error reading file: %s" % err)
sys.exit(1)
try:
tree = compile(
source,
filename="<source>",
mode="exec",
flags=ast.PyCF_ONLY_AST,
dont_inherit=True,
optimize=0,
)
except SyntaxError as err:
# Output the syntax error as a single entry so CE can display it
print("SyntaxError: %s" % err)
sys.exit(1)
# The "ident" parameter is introduced in Python 3.9
print(ast.dump(tree, indent=2))
if __name__ == "__main__":
main()

37
lib/compilers/python.ts
View File

@@ -24,15 +24,20 @@
// POSSIBILITY OF SUCH DAMAGE.
import type {AsmResultSource, ParsedAsmResultLine} from '../../types/asmresult/asmresult.interfaces.js';
import type {CompilationResult} from '../../types/compilation/compilation.interfaces.js';
import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js';
import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js';
import type {ResultLine} from '../../types/resultline/resultline.interfaces.js';
import {BaseCompiler} from '../base-compiler.js';
import {CompilationEnvironment} from '../compilation-env.js';
import {PythonAstParser} from '../python-ast.js';
import {resolvePathFromAppRoot} from '../utils.js';
import {BaseParser} from './argument-parsers.js';
export class PythonCompiler extends BaseCompiler {
private readonly disasmScriptPath: string;
private readonly astScriptPath: string;
private readonly pythonAstParser: PythonAstParser;
static get key() {
return 'python';
@@ -45,6 +50,9 @@ export class PythonCompiler extends BaseCompiler {
this.disasmScriptPath =
this.compilerProps<string>('disasmScript') ||
resolvePathFromAppRoot('etc', 'scripts', 'disasms', 'dis_all.py');
this.astScriptPath =
this.compilerProps<string>('astScript') || resolvePathFromAppRoot('etc', 'scripts', 'ast_dump.py');
this.pythonAstParser = new PythonAstParser(this.compilerProps);
}
override async processAsm(result) {
@@ -80,6 +88,35 @@ export class PythonCompiler extends BaseCompiler {
return ['-I', this.disasmScriptPath, '--outputfile', outputFilename, '--inputfile'];
}
override couldSupportASTDump(version: string) {
// Python's ast module is available since Python 2.6. However, the
// formatted dump of Python AST via `ast.dump()` is available since
// Python 3.9. Therefore, we return true for all versions >= 3.9.
const versionRegex = /Python (\d+)\.(\d+)\.(\d+)$/;
const versionMatch = versionRegex.exec(version);
if (versionMatch === null) {
return false;
}
const major = Number.parseInt(versionMatch[1]);
const minor = Number.parseInt(versionMatch[2]);
return major > 3 || (major === 3 && minor >= 9);
}
override async generateAST(inputFilename: string, options: string[]): Promise<ResultLine[]> {
const astOptions = ['-I', this.astScriptPath, this.filename(inputFilename)];
const execOptions = this.getDefaultExecOptions();
const result: CompilationResult = await this.runCompiler(
this.compiler.exe,
astOptions,
this.filename(inputFilename),
execOptions,
);
return this.pythonAstParser.processAst(result);
}
override getArgumentParserClass() {
return BaseParser;
}

43
lib/python-ast.ts Normal file
View File

@@ -0,0 +1,43 @@
// Copyright (c) 2026, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
import type {CompilationResult} from '../types/compilation/compilation.interfaces.js';
import type {ResultLine} from '../types/resultline/resultline.interfaces.js';
import type {PropertyGetter} from './properties.interfaces.js';
export class PythonAstParser {
private readonly maxAstLines: number;
constructor(compilerProps: PropertyGetter) {
this.maxAstLines = 500;
if (compilerProps) {
this.maxAstLines = compilerProps('maxLinesOfAst', this.maxAstLines);
}
}
processAst(result: CompilationResult): ResultLine[] {
const sliceEnd = Math.min(this.maxAstLines, result.stdout.length);
return result.stdout.slice(0, sliceEnd);
}
}

160
test/ast/python-long.ast Normal file
View File

@@ -0,0 +1,160 @@
Module(
body=[
Expr(
value=Constant(value='Dumps the AST of a Python source file as structured JSON for Compiler Explorer.')),
Import(
names=[
alias(name='argparse')]),
Import(
names=[
alias(name='ast')]),
Import(
names=[
alias(name='sys')]),
FunctionDef(
name='main',
args=arguments(),
body=[
Assign(
targets=[
Name(id='parser', ctx=Store())],
value=Call(
func=Attribute(
value=Name(id='argparse', ctx=Load()),
attr='ArgumentParser',
ctx=Load()),
keywords=[
keyword(
arg='description',
value=Constant(value='Parse and dump the AST of an Python source file'))])),
Expr(
value=Call(
func=Attribute(
value=Name(id='parser', ctx=Load()),
attr='add_argument',
ctx=Load()),
args=[
Constant(value='input')],
keywords=[
keyword(
arg='help',
value=Constant(value='Path to input Python source code file'))])),
Assign(
targets=[
Name(id='args', ctx=Store())],
value=Call(
func=Attribute(
value=Name(id='parser', ctx=Load()),
attr='parse_args',
ctx=Load()))),
Try(
body=[
With(
items=[
withitem(
context_expr=Call(
func=Name(id='open', ctx=Load()),
args=[
Attribute(
value=Name(id='args', ctx=Load()),
attr='input',
ctx=Load()),
Constant(value='r')],
keywords=[
keyword(
arg='encoding',
value=Constant(value='utf-8'))]),
optional_vars=Name(id='f', ctx=Store()))],
body=[
Assign(
targets=[
Name(id='source', ctx=Store())],
value=Call(
func=Attribute(
value=Name(id='f', ctx=Load()),
attr='read',
ctx=Load())))])],
handlers=[
ExceptHandler(
type=Name(id='IOError', ctx=Load()),
name='err',
body=[
Expr(
value=Call(
func=Name(id='print', ctx=Load()),
args=[
BinOp(
left=Constant(value='Error reading file: %s'),
op=Mod(),
right=Name(id='err', ctx=Load()))])),
Expr(
value=Call(
func=Attribute(
value=Name(id='sys', ctx=Load()),
attr='exit',
ctx=Load()),
args=[
Constant(value=1)]))])]),
Try(
body=[
Assign(
targets=[
Name(id='tree', ctx=Store())],
value=Call(
func=Attribute(
value=Name(id='ast', ctx=Load()),
attr='parse',
ctx=Load()),
args=[
Name(id='source', ctx=Load())],
keywords=[
keyword(
arg='optimize',
value=Constant(value=0))]))],
handlers=[
ExceptHandler(
type=Name(id='SyntaxError', ctx=Load()),
name='err',
body=[
Expr(
value=Call(
func=Name(id='print', ctx=Load()),
args=[
BinOp(
left=Constant(value='SyntaxError: %s'),
op=Mod(),
right=Name(id='err', ctx=Load()))])),
Expr(
value=Call(
func=Attribute(
value=Name(id='sys', ctx=Load()),
attr='exit',
ctx=Load()),
args=[
Constant(value=1)]))])]),
Expr(
value=Call(
func=Name(id='print', ctx=Load()),
args=[
Call(
func=Attribute(
value=Name(id='ast', ctx=Load()),
attr='dump',
ctx=Load()),
args=[
Name(id='tree', ctx=Load())],
keywords=[
keyword(
arg='indent',
value=Constant(value=2))])]))]),
If(
test=Compare(
left=Name(id='__name__', ctx=Load()),
ops=[
Eq()],
comparators=[
Constant(value='__main__')]),
body=[
Expr(
value=Call(
func=Name(id='main', ctx=Load())))])])

13
test/ast/python-short.ast Normal file
View File

@@ -0,0 +1,13 @@
Module(
body=[
FunctionDef(
name='square',
args=arguments(
args=[
arg(arg='num')]),
body=[
Return(
value=BinOp(
left=Name(id='num', ctx=Load()),
op=Mult(),
right=Name(id='num', ctx=Load())))])])

71
test/python-ast-parser-tests.ts Normal file
View File

@@ -0,0 +1,71 @@
// Copyright (c) 2026, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
import fs from 'node:fs';
import {beforeAll, describe, expect, it} from 'vitest';
import * as properties from '../lib/properties.js';
import {PythonAstParser} from '../lib/python-ast.js';
import * as utils from '../lib/utils.js';
const languages = {
python: {id: 'python'},
};
function mockAstOutput(astLines: string[]) {
return {stdout: astLines.map(l => ({text: l}))};
}
describe('python-ast', () => {
let compilerProps;
let astParser: any;
let shortAstDump: string[];
let longAstDump: string[];
beforeAll(() => {
const fakeProps = new properties.CompilerProps(
languages,
properties.fakeProps({
maxLinesOfAst: 100,
}),
);
compilerProps = (fakeProps.get as any).bind(fakeProps, 'python');
astParser = new PythonAstParser(compilerProps);
shortAstDump = utils.splitLines(fs.readFileSync('test/ast/python-short.ast').toString());
longAstDump = utils.splitLines(fs.readFileSync('test/ast/python-long.ast').toString());
});
it('keeps original lines on short ast output', () => {
const processedLines = astParser.processAst(mockAstOutput(shortAstDump));
expect(processedLines.length).toBe(shortAstDump.length);
});
it('truncates long ast output', () => {
const processedLines = astParser.processAst(mockAstOutput(longAstDump));
expect(processedLines.length).toBe(100);
});
});