From 7efe253f5fffd306c866d92944e97342f4acaf99 Mon Sep 17 00:00:00 2001 From: kevinjeon-g <136382173+kevinjeon-g@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:35:56 -0500 Subject: [PATCH] Add support for Android D8 (8.1.56) (#5756) Adds D8Compiler, which applies to the Android Java and Android Kotlin languages. D8Compiler instantiates a JavaCompiler or KotlinCompiler using the java/kotlin dependencies' paths for D8 in the infra repo. compiler-finder.ts has been updated to allow for duplicate compiler IDs for 'android-java' and 'android-kotlin', as it is expected that the compilers used for these languages is the same. --- .github/labeler.yml | 6 + app.ts | 10 +- etc/config/android-java.amazon.properties | 15 ++ etc/config/android-java.defaults.properties | 17 ++ etc/config/android-kotlin.amazon.properties | 15 ++ etc/config/android-kotlin.defaults.properties | 17 ++ examples/android-java/default.java | 7 + examples/android-kotlin/default.kt | 2 + lib/compiler-finder.ts | 6 +- lib/compilers/_all.ts | 1 + lib/compilers/d8.ts | 214 ++++++++++++++++++ lib/languages.ts | 22 ++ types/languages.interfaces.ts | 2 + views/resources/logos/android-dark.svg | 27 +++ views/resources/logos/android.svg | 27 +++ 15 files changed, 384 insertions(+), 4 deletions(-) create mode 100644 etc/config/android-java.amazon.properties create mode 100644 etc/config/android-java.defaults.properties create mode 100644 etc/config/android-kotlin.amazon.properties create mode 100644 etc/config/android-kotlin.defaults.properties create mode 100644 examples/android-java/default.java create mode 100644 examples/android-kotlin/default.kt create mode 100644 lib/compilers/d8.ts create mode 100644 views/resources/logos/android-dark.svg create mode 100644 views/resources/logos/android.svg diff --git a/.github/labeler.yml b/.github/labeler.yml index bac169324..1f48a213e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,6 +5,12 @@ lang-ada: - lib/compilers/ada.ts - etc/config/ada.*.properties - static/modes/ada-mode.ts +lang-android-java: + - lib/compilers/d8.ts + - etc/config/android-java.*.properties +lang-android-kotlin: + - lib/compilers/d8.ts + - etc/config/android-kotlin.*.properties lang-asm: - lib/compilers/assembly.ts - lib/compilers/nasm.ts diff --git a/app.ts b/app.ts index cff90c01c..31a8fc7bc 100755 --- a/app.ts +++ b/app.ts @@ -73,6 +73,9 @@ import type {Language, LanguageKey} from './types/languages.interfaces.js'; // Used by assert.ts global.ce_base_directory = new URL('.', import.meta.url); +// Used by d8.ts +global.handlerConfig = null; + (nopt as any).invalidHandler = (key, val, types) => { logger.error( `Command line argument type error for "--${key}=${val}", expected ${types.map(t => typeof t).join(' | ')}`, @@ -562,7 +565,8 @@ async function main() { const healthCheckFilePath = ceProps('healthCheckFilePath', false); - const handlerConfig = { + // Exported to allow compilers to refer to other existing compilers. + global.handlerConfig = { compileHandler, clientOptionsHandler, storageHandler, @@ -575,8 +579,8 @@ async function main() { contentPolicyHeader, }; - const noscriptHandler = new NoScriptHandler(router, handlerConfig); - const routeApi = new RouteAPI(router, handlerConfig); + const noscriptHandler = new NoScriptHandler(router, global.handlerConfig); + const routeApi = new RouteAPI(router, global.handlerConfig); async function onCompilerChange(compilers) { if (JSON.stringify(prevCompilers) === JSON.stringify(compilers)) { diff --git a/etc/config/android-java.amazon.properties b/etc/config/android-java.amazon.properties new file mode 100644 index 000000000..84eeaee12 --- /dev/null +++ b/etc/config/android-java.amazon.properties @@ -0,0 +1,15 @@ +compilers=&android-d8 + +group.android-d8.compilers=d8-8156 +group.android-d8.compilerType=android-d8 +group.android-d8.objdumper=/opt/compiler-explorer/baksmali-3.0.3/baksmali-3.0.3-fat.jar + +compiler.d8-8156.name=d8 8.1.56 +compiler.d8-8156.exe=/opt/compiler-explorer/r8-8.1.56/r8-8.1.56.jar +compiler.d8-8156.javaId=java1601 +compiler.d8-8156.javaPath=/opt/compiler-explorer/jdk-16.0.1/bin/java +compiler.d8-8156.javacPath=/opt/compiler-explorer/jdk-16.0.1/bin/javac +compiler.d8-8156.kotlinId=kotlinc1920 +compiler.d8-8156.kotlincPath=/opt/compiler-explorer/kotlin-jvm-1.9.20/bin/kotlinc + +defaultCompiler=d8-8156 diff --git a/etc/config/android-java.defaults.properties b/etc/config/android-java.defaults.properties new file mode 100644 index 000000000..aa9d5a02e --- /dev/null +++ b/etc/config/android-java.defaults.properties @@ -0,0 +1,17 @@ +compilers=&android-d8 +compilerType=android-d8 +objdumper=/usr/local/bin/baksmali.jar + +group.android-d8.compilers=android-d8-default +compiler.android-d8-default.name=android d8 default +compiler.android-d8-default.exe=/usr/local/bin/r8.jar + +# When testing locally, these paths must be valid and should +# reflect the paths in the java and kotlin default config. +compiler.android-d8-default.javaId=javacdefault +compiler.android-d8-default.javaPath=/usr/bin/java +compiler.android-d8-default.javacPath=/usr/bin/javac +compiler.android-d8-default.kotlinId=kotlincdefault +compiler.android-d8-default.kotlincPath=/usr/bin/kotlinc-jvm + +defaultCompiler=android-d8-default diff --git a/etc/config/android-kotlin.amazon.properties b/etc/config/android-kotlin.amazon.properties new file mode 100644 index 000000000..84eeaee12 --- /dev/null +++ b/etc/config/android-kotlin.amazon.properties @@ -0,0 +1,15 @@ +compilers=&android-d8 + +group.android-d8.compilers=d8-8156 +group.android-d8.compilerType=android-d8 +group.android-d8.objdumper=/opt/compiler-explorer/baksmali-3.0.3/baksmali-3.0.3-fat.jar + +compiler.d8-8156.name=d8 8.1.56 +compiler.d8-8156.exe=/opt/compiler-explorer/r8-8.1.56/r8-8.1.56.jar +compiler.d8-8156.javaId=java1601 +compiler.d8-8156.javaPath=/opt/compiler-explorer/jdk-16.0.1/bin/java +compiler.d8-8156.javacPath=/opt/compiler-explorer/jdk-16.0.1/bin/javac +compiler.d8-8156.kotlinId=kotlinc1920 +compiler.d8-8156.kotlincPath=/opt/compiler-explorer/kotlin-jvm-1.9.20/bin/kotlinc + +defaultCompiler=d8-8156 diff --git a/etc/config/android-kotlin.defaults.properties b/etc/config/android-kotlin.defaults.properties new file mode 100644 index 000000000..aa9d5a02e --- /dev/null +++ b/etc/config/android-kotlin.defaults.properties @@ -0,0 +1,17 @@ +compilers=&android-d8 +compilerType=android-d8 +objdumper=/usr/local/bin/baksmali.jar + +group.android-d8.compilers=android-d8-default +compiler.android-d8-default.name=android d8 default +compiler.android-d8-default.exe=/usr/local/bin/r8.jar + +# When testing locally, these paths must be valid and should +# reflect the paths in the java and kotlin default config. +compiler.android-d8-default.javaId=javacdefault +compiler.android-d8-default.javaPath=/usr/bin/java +compiler.android-d8-default.javacPath=/usr/bin/javac +compiler.android-d8-default.kotlinId=kotlincdefault +compiler.android-d8-default.kotlincPath=/usr/bin/kotlinc-jvm + +defaultCompiler=android-d8-default diff --git a/examples/android-java/default.java b/examples/android-java/default.java new file mode 100644 index 000000000..811998578 --- /dev/null +++ b/examples/android-java/default.java @@ -0,0 +1,7 @@ +// Type your code here, or load an example. +class Square { + static int square(int num) { + return num * num; + } +} + diff --git a/examples/android-kotlin/default.kt b/examples/android-kotlin/default.kt new file mode 100644 index 000000000..b1576c436 --- /dev/null +++ b/examples/android-kotlin/default.kt @@ -0,0 +1,2 @@ +// Type your code here, or load an example. +fun square(num: Int): Int = num * num diff --git a/lib/compiler-finder.ts b/lib/compiler-finder.ts index c4b613f50..f2b2c42dc 100644 --- a/lib/compiler-finder.ts +++ b/lib/compiler-finder.ts @@ -536,7 +536,11 @@ export class CompilerFinder { logger.error( `Duplicate compiler id ${propParts[1]} in domain ${domains.join(',')}`, ); - error = true; + // android-java and android-kotlin are + // expected to use the exact same compilers. + if (lang !== 'android-java' && lang !== 'android-kotlin') { + error = true; + } } compilers.add(propParts[1]); } diff --git a/lib/compilers/_all.ts b/lib/compilers/_all.ts index 36e5ac8d1..16b722bf1 100644 --- a/lib/compilers/_all.ts +++ b/lib/compilers/_all.ts @@ -47,6 +47,7 @@ export {CprocCompiler} from './cproc.js'; export {CLSPVCompiler} from './clspv.js'; export {CrystalCompiler} from './crystal.js'; export {CSharpCompiler} from './dotnet.js'; +export {D8Compiler} from './d8.js'; export {DartCompiler} from './dart.js'; export {DefaultCompiler} from './default.js'; export {DMDCompiler} from './dmd.js'; diff --git a/lib/compilers/d8.ts b/lib/compilers/d8.ts new file mode 100644 index 000000000..89d3383bb --- /dev/null +++ b/lib/compilers/d8.ts @@ -0,0 +1,214 @@ +// Copyright (c) 2023, Compiler Explorer Authors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +import path from 'path'; + +import fs from 'fs-extra'; +import _ from 'underscore'; + +import {logger} from '../logger.js'; + +import {BaseCompiler} from '../base-compiler.js'; + +import type {PreliminaryCompilerInfo} from '../../types/compiler.interfaces.js'; +import type {ParsedAsmResult, ParsedAsmResultLine} from '../../types/asmresult/asmresult.interfaces.js'; +import {CompilationResult, ExecutionOptions} from '../../types/compilation/compilation.interfaces.js'; +import type {ParseFiltersAndOutputOptions} from '../../types/features/filters.interfaces.js'; + +export class D8Compiler extends BaseCompiler { + static get key() { + return 'android-d8'; + } + + lineNumberRegex: RegExp; + methodEndRegex: RegExp; + + javaId: string; + kotlinId: string; + + javaPath: string; + javacPath: string; + kotlincPath: string; + + constructor(compilerInfo: PreliminaryCompilerInfo, env) { + super({...compilerInfo}, env); + + this.lineNumberRegex = /^\s+\.line\s+(\d+).*$/; + this.methodEndRegex = /^\s*\.end\smethod.*$/; + + this.javaId = this.compilerProps(`compiler.${this.compiler.id}.javaId`); + this.kotlinId = this.compilerProps(`compiler.${this.compiler.id}.kotlinId`); + + this.javaPath = this.compilerProps(`compiler.${this.compiler.id}.javaPath`); + this.javacPath = this.compilerProps(`compiler.${this.compiler.id}.javacPath`); + this.kotlincPath = this.compilerProps(`compiler.${this.compiler.id}.kotlincPath`); + } + + override getOutputFilename(dirPath: string) { + return path.join(dirPath, `${path.basename(this.compileFilename, this.lang.extensions[0])}.dex`); + } + + override async runCompiler( + compiler: string, + options: string[], + inputFilename: string, + execOptions: ExecutionOptions & {env: Record}, + ): Promise { + const preliminaryCompilePath = path.dirname(inputFilename); + let outputFilename = ''; + + // Instantiate Java or Kotlin compiler based on the current language. + if (this.lang.id === 'android-java') { + const javaCompiler = global.handlerConfig.compileHandler.findCompiler('java', this.javaId); + outputFilename = javaCompiler.getOutputFilename(preliminaryCompilePath); + const javaOptions = _.compact( + javaCompiler.prepareArguments( + [''], // options + javaCompiler.getDefaultFilters(), + {}, // backendOptions + inputFilename, + outputFilename, + [], // libraries + [], // overrides + ), + ); + await javaCompiler.runCompiler( + this.javacPath, + javaOptions, + this.filename(inputFilename), + javaCompiler.getDefaultExecOptions(), + ); + } else if (this.lang.id === 'android-kotlin') { + const kotlinCompiler = global.handlerConfig.compileHandler.findCompiler('kotlin', this.kotlinId); + outputFilename = kotlinCompiler.getOutputFilename(preliminaryCompilePath); + const kotlinOptions = _.compact( + kotlinCompiler.prepareArguments( + [''], // options + kotlinCompiler.getDefaultFilters(), + {}, // backendOptions + inputFilename, + outputFilename, + [], // libraries + [], // overrides + ), + ); + await kotlinCompiler.runCompiler( + this.kotlincPath, + kotlinOptions, + this.filename(inputFilename), + kotlinCompiler.getDefaultExecOptions(), + ); + } else { + logger.error('Language is neither android-java nor android-kotlin.'); + } + + if (!execOptions) { + execOptions = this.getDefaultExecOptions(); + } + if (!execOptions.customCwd) { + execOptions.customCwd = path.dirname(inputFilename); + } + + // The items in 'options' before the source file are user inputs. + const sourceFileOptionIndex = options.findIndex(option => { + return option.endsWith('.java') || option.endsWith('.kt'); + }); + + const files = await fs.readdir(preliminaryCompilePath); + const classFiles = files.filter(f => f.endsWith('.class')); + const d8Options = [ + '-cp', + this.compiler.exe, // R8 jar. + 'com.android.tools.r8.D8', // Main class name for the D8 compiler. + ...options.slice(0, sourceFileOptionIndex), + ...classFiles, + ]; + const result = await this.exec(this.javaPath, d8Options, execOptions); + return { + ...this.transformToCompilationResult(result, outputFilename), + languageId: this.getCompilerResultLanguageId(), + }; + } + + override async objdump(outputFilename, result: any, maxSize: number) { + const dirPath = path.dirname(outputFilename); + + // There is only one dex file for all classes. + let files = await fs.readdir(dirPath); + const dexFile = files.find(f => f.endsWith('.dex')); + const baksmaliOptions = ['-jar', this.compiler.objdumper, 'd', `${dexFile}`, '-o', dirPath]; + const baksmaliResult = await this.exec(this.javaPath, baksmaliOptions, { + maxOutput: maxSize, + customCwd: dirPath, + }); + + // There is one smali file for each class. + files = await fs.readdir(dirPath); + const smaliFiles = files.filter(f => f.endsWith('.smali')); + let objResult = ''; + for (const smaliFile of smaliFiles) { + objResult = objResult.concat(fs.readFileSync(path.join(dirPath, smaliFile), 'utf-8') + '\n\n'); + } + + const asmResult: ParsedAsmResult = { + asm: [ + { + text: objResult, + }, + ], + }; + + result.asm = asmResult.asm; + return result; + } + + override optionsForFilter(filters: ParseFiltersAndOutputOptions) { + filters.binary = true; + return []; + } + + // Map line numbers to lines. + override async processAsm(result) { + const asm = result.asm[0].text; + const segments: ParsedAsmResultLine[] = []; + + let lineNumber; + for (const l of asm.split(/\n/)) { + if (this.lineNumberRegex.test(l)) { + lineNumber = Number.parseInt(l.match(this.lineNumberRegex)[1]); + segments.push({text: l, source: null}); + } else if (this.methodEndRegex.test(l)) { + lineNumber = null; + segments.push({text: l, source: null}); + } else { + if (/\S/.test(l) && lineNumber) { + segments.push({text: l, source: {file: null, line: lineNumber}}); + } else { + segments.push({text: l, source: null}); + } + } + } + return {asm: segments}; + } +} diff --git a/lib/languages.ts b/lib/languages.ts index 05642c8c0..0db77299a 100644 --- a/lib/languages.ts +++ b/lib/languages.ts @@ -87,6 +87,28 @@ const definitions: Record = { monacoDisassembly: null, tooltip: 'A collection of asm analysis tools', }, + 'android-java': { + name: 'Android Java', + monaco: 'java', + extensions: ['.java'], + alias: [], + logoUrl: 'android.svg', + logoUrlDark: 'android-dark.svg', + formatter: null, + previewFilter: null, + monacoDisassembly: null, + }, + 'android-kotlin': { + name: 'Android Kotlin', + monaco: 'kotlin', + extensions: ['.kt'], + alias: [], + logoUrl: 'android.svg', + logoUrlDark: 'android-dark.svg', + formatter: null, + previewFilter: null, + monacoDisassembly: null, + }, assembly: { name: 'Assembly', monaco: 'asm', diff --git a/types/languages.interfaces.ts b/types/languages.interfaces.ts index af3b6e7c3..e64871ac4 100644 --- a/types/languages.interfaces.ts +++ b/types/languages.interfaces.ts @@ -25,6 +25,8 @@ export type LanguageKey = | 'ada' | 'analysis' + | 'android-java' + | 'android-kotlin' | 'assembly' | 'c' | 'c++' diff --git a/views/resources/logos/android-dark.svg b/views/resources/logos/android-dark.svg new file mode 100644 index 000000000..1ce2fc6ee --- /dev/null +++ b/views/resources/logos/android-dark.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + diff --git a/views/resources/logos/android.svg b/views/resources/logos/android.svg new file mode 100644 index 000000000..1cdb94b2e --- /dev/null +++ b/views/resources/logos/android.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + +