From 3563f5314a64f5133d19fe29a4379ff171c4102b Mon Sep 17 00:00:00 2001 From: djole Date: Wed, 13 May 2026 12:14:50 +0200 Subject: [PATCH 1/3] test: add ethdebug/conformance --- .github/workflows/ethdebug.yml | 137 ++++++++++ .prettierignore | 3 + package.json | 2 +- packages/conformance/README.md | 41 +++ packages/conformance/package.json | 57 ++++ packages/conformance/src/adapters/anvil.ts | 250 ++++++++++++++++++ packages/conformance/src/adapters/bugc.ts | 122 +++++++++ packages/conformance/src/adapters/solc.ts | 150 +++++++++++ packages/conformance/src/adapters/soldb.ts | 128 +++++++++ packages/conformance/src/index.ts | 6 + packages/conformance/src/runner.ts | 147 ++++++++++ packages/conformance/src/types.ts | 90 +++++++ packages/conformance/test/conformance.test.ts | 201 ++++++++++++++ .../test/fixtures/bugc/minimal.bug | 12 + .../test/fixtures/solc/Counter.sol | 10 + .../fixtures/solc/multi-source/Counter.sol | 12 + .../test/fixtures/solc/multi-source/Math.sol | 8 + packages/conformance/tsconfig.json | 19 ++ packages/conformance/vitest.config.ts | 6 + 19 files changed, 1400 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ethdebug.yml create mode 100644 packages/conformance/README.md create mode 100644 packages/conformance/package.json create mode 100644 packages/conformance/src/adapters/anvil.ts create mode 100644 packages/conformance/src/adapters/bugc.ts create mode 100644 packages/conformance/src/adapters/solc.ts create mode 100644 packages/conformance/src/adapters/soldb.ts create mode 100644 packages/conformance/src/index.ts create mode 100644 packages/conformance/src/runner.ts create mode 100644 packages/conformance/src/types.ts create mode 100644 packages/conformance/test/conformance.test.ts create mode 100644 packages/conformance/test/fixtures/bugc/minimal.bug create mode 100644 packages/conformance/test/fixtures/solc/Counter.sol create mode 100644 packages/conformance/test/fixtures/solc/multi-source/Counter.sol create mode 100644 packages/conformance/test/fixtures/solc/multi-source/Math.sol create mode 100644 packages/conformance/tsconfig.json create mode 100644 packages/conformance/vitest.config.ts diff --git a/.github/workflows/ethdebug.yml b/.github/workflows/ethdebug.yml new file mode 100644 index 000000000..c9d02af34 --- /dev/null +++ b/.github/workflows/ethdebug.yml @@ -0,0 +1,137 @@ +name: ETHDebug + +on: + pull_request: + workflow_dispatch: + inputs: + solidity_ref: + description: Solidity ref to build for solc + default: feature/ethdebug + required: true + soldb_ref: + description: SolDB ref to build + default: main + required: true + +concurrency: + group: ethdebug-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FOUNDRY_VERSION: v1.0.0 + SOLIDITY_REF: ${{ github.event.inputs.solidity_ref || 'feature/ethdebug' }} + SOLDB_REF: ${{ github.event.inputs.soldb_ref || 'main' }} + +jobs: + conformance: + name: solc + soldb conformance + runs-on: ubuntu-24.04 + + steps: + - name: Checkout ethdebug/format + uses: actions/checkout@v4 + with: + path: format + + - name: Checkout Solidity + uses: actions/checkout@v4 + with: + repository: walnuthq/solidity + ref: ${{ env.SOLIDITY_REF }} + path: solidity + submodules: recursive + + - name: Checkout SolDB + uses: actions/checkout@v4 + with: + repository: walnuthq/soldb + ref: ${{ env.SOLDB_REF }} + path: soldb + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: yarn + cache-dependency-path: format/yarn.lock + + - name: Install format dependencies + run: yarn install --frozen-lockfile + working-directory: format + + - name: Install Rust toolchain + run: | + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Cache Cargo build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/git + ~/.cargo/registry + soldb/target + key: ${{ runner.os }}-soldb-cargo-${{ hashFiles('soldb/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-soldb-cargo- + + - name: Build SolDB + run: cargo build --bin soldb + working-directory: soldb + + - name: Install Solidity build dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + ccache \ + cmake \ + libboost-filesystem-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-test-dev \ + libcln-dev \ + ninja-build + + - name: Cache Solidity ccache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-solidity-ccache-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-solidity-ccache- + + - name: Build solc with ETHDebug support + run: | + cmake \ + -S solidity \ + -B solidity/build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DTESTS=OFF \ + -DTOOLS=OFF \ + -DPEDANTIC=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build solidity/build --target solc --parallel 2 + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + with: + version: ${{ env.FOUNDRY_VERSION }} + + - name: Show external tool versions + run: | + solidity/build/solc/solc --version + soldb/target/debug/soldb --version + anvil --version + cast --version + + - name: Run ETHDebug conformance tests + env: + ETHDEBUG_CONFORMANCE_SOLC: ${{ github.workspace }}/solidity/build/solc/solc + ETHDEBUG_CONFORMANCE_SOLDB: ${{ github.workspace }}/soldb/target/debug/soldb + ETHDEBUG_CONFORMANCE_ANVIL: anvil + ETHDEBUG_CONFORMANCE_CAST: cast + run: yarn --cwd packages/conformance test:external + working-directory: format diff --git a/.prettierignore b/.prettierignore index ad23bcd4d..5ed4b7aad 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,6 +9,9 @@ package-lock.json # Auto-generated files packages/format/src/schemas/yamls.ts +# Solidity fixtures are compiler inputs; this repo has no Solidity Prettier parser. +packages/conformance/test/fixtures/solc/**/*.sol + # Docusaurus build output packages/web/.docusaurus/ packages/web/build/ diff --git a/package.json b/package.json index 9e55b2c7e..58c4647fb 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "packages/*" ], "scripts": { - "build": "yarn --cwd packages/format prepare:yamls && tsc --build packages/format packages/pointers packages/evm packages/bugc packages/programs-react packages/pointers-react", + "build": "yarn --cwd packages/format prepare:yamls && tsc --build packages/format packages/pointers packages/evm packages/bugc packages/conformance packages/programs-react packages/pointers-react", "bundle": "tsx ./bin/bundle-schema.ts", "test": "vitest", "test:coverage": "vitest run --coverage", diff --git a/packages/conformance/README.md b/packages/conformance/README.md new file mode 100644 index 000000000..8aeeef8b6 --- /dev/null +++ b/packages/conformance/README.md @@ -0,0 +1,41 @@ +# @ethdebug/conformance + +Reusable ETHDebug conformance infrastructure. + +This package is intentionally language- and consumer-neutral. Compiler adapters +produce ETHDebug artifacts; consumer adapters exercise debugger implementations +against those artifacts. + +Current adapters: + +- `bugc`: in-process BUG compiler adapter. +- `solc`: external `solc --standard-json` adapter. +- `soldb`: external SolDB CLI adapter. + +The first layer checks the contract that every compiler should satisfy: + +- emitted programs are valid `ethdebug/format/program` objects, +- resources and compilations are valid when present, +- source references used by programs resolve to compilation sources. + +The second layer is Dexter-like consumer conformance: tests can run a debugger +consumer, parse its output, and assert resources, source steps, frames, or +values. SolDB is the first consumer backend, but the runner is not tied to +SolDB. The SolDB adapter can materialize compiler output as a SolDB-compatible +debug directory and then drive `soldb info resources` over it. The optional +Foundry adapter starts a local `anvil --steps-tracing` node, deploys a compiled +contract with `cast`, sends a transaction, and scripts SolDB's interactive REPL +to assert source-line breakpoint set/hit behavior. + +External adapters are opt-in in tests: + +```console +ETHDEBUG_CONFORMANCE_SOLC=/path/to/solc yarn test +ETHDEBUG_CONFORMANCE_SOLDB=/path/to/soldb yarn test +ETHDEBUG_CONFORMANCE_ANVIL=/path/to/anvil ETHDEBUG_CONFORMANCE_CAST=/path/to/cast yarn test +``` + +To run the full Solidity -> SolDB resources path, set `ETHDEBUG_CONFORMANCE_SOLC` +and `ETHDEBUG_CONFORMANCE_SOLDB`. If `anvil` and `cast` are on `PATH`, the +SolDB breakpoint test runs as well; the executable paths can be overridden with +`ETHDEBUG_CONFORMANCE_ANVIL` and `ETHDEBUG_CONFORMANCE_CAST`. diff --git a/packages/conformance/package.json b/packages/conformance/package.json new file mode 100644 index 000000000..2812f3ff6 --- /dev/null +++ b/packages/conformance/package.json @@ -0,0 +1,57 @@ +{ + "name": "@ethdebug/conformance", + "version": "0.1.0-0", + "description": "Reusable ETHDebug conformance runner and adapters", + "type": "module", + "main": "dist/src/index.js", + "types": "dist/src/index.d.ts", + "license": "MIT", + "files": [ + "dist", + "README.md" + ], + "imports": { + "#adapters/anvil": { + "types": "./src/adapters/anvil.ts", + "default": "./dist/src/adapters/anvil.js" + }, + "#adapters/bugc": { + "types": "./src/adapters/bugc.ts", + "default": "./dist/src/adapters/bugc.js" + }, + "#adapters/soldb": { + "types": "./src/adapters/soldb.ts", + "default": "./dist/src/adapters/soldb.js" + }, + "#adapters/solc": { + "types": "./src/adapters/solc.ts", + "default": "./dist/src/adapters/solc.js" + }, + "#runner": { + "types": "./src/runner.ts", + "default": "./dist/src/runner.js" + }, + "#types": { + "types": "./src/types.ts", + "default": "./dist/src/types.js" + } + }, + "scripts": { + "prepare": "tsc", + "build": "tsc", + "test": "vitest run", + "test:external": "vitest run --no-file-parallelism --maxWorkers=1" + }, + "dependencies": { + "@ethdebug/bugc": "^0.1.0-0", + "@ethdebug/format": "^0.1.0-0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0", + "vitest": "^3.2.4" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/conformance/src/adapters/anvil.ts b/packages/conformance/src/adapters/anvil.ts new file mode 100644 index 000000000..f57b20756 --- /dev/null +++ b/packages/conformance/src/adapters/anvil.ts @@ -0,0 +1,250 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { createServer, type AddressInfo } from "node:net"; +import { setTimeout as delay } from "node:timers/promises"; + +const DEFAULT_HOST = "127.0.0.1"; +const DEFAULT_PRIVATE_KEY = + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + +export interface AnvilOptions { + executable?: string; + host?: string; + port?: number; + stepsTracing?: boolean; + silent?: boolean; +} + +export interface AnvilInstance { + rpcUrl: string; + stop(): Promise; +} + +export interface CastOptions { + executable?: string; + privateKey?: string; +} + +export interface TransactionReceipt { + transactionHash: string; + contractAddress?: string | null; +} + +async function freePort(host: string): Promise { + return await new Promise((resolve, reject) => { + const server = createServer(); + server.on("error", reject); + server.listen(0, host, () => { + const address = server.address() as AddressInfo; + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(address.port); + } + }); + }); + }); +} + +async function rpcRequest(rpcUrl: string, method: string): Promise { + const response = await fetch(rpcUrl, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method, + params: [], + }), + }); + if (!response.ok) { + throw new Error(`RPC ${method} failed with HTTP ${response.status}`); + } + + const body = (await response.json()) as { error?: unknown; result?: unknown }; + if (body.error) { + throw new Error(`RPC ${method} failed: ${JSON.stringify(body.error)}`); + } + return body.result; +} + +async function waitForRpc( + child: ChildProcess, + rpcUrl: string, + stderr: () => string, +): Promise { + const deadline = Date.now() + 10_000; + let spawnError: Error | undefined; + child.once("error", (error) => { + spawnError = error; + }); + + while (Date.now() < deadline) { + if (spawnError) { + throw spawnError; + } + if (child.exitCode !== null) { + throw new Error( + `anvil exited before accepting RPC requests\n${stderr()}`, + ); + } + + try { + await rpcRequest(rpcUrl, "eth_chainId"); + return; + } catch { + await delay(100); + } + } + + throw new Error(`timed out waiting for anvil at ${rpcUrl}\n${stderr()}`); +} + +export async function startAnvil( + options: AnvilOptions = {}, +): Promise { + const executable = + options.executable ?? process.env.ETHDEBUG_CONFORMANCE_ANVIL ?? "anvil"; + const host = options.host ?? DEFAULT_HOST; + const port = options.port ?? (await freePort(host)); + const rpcUrl = `http://${host}:${port}`; + const args = ["--host", host, "--port", String(port)]; + + if (options.stepsTracing ?? true) { + args.push("--steps-tracing"); + } + if (options.silent ?? true) { + args.push("--silent"); + } + + const child = spawn(executable, args, { + stdio: ["ignore", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk) => { + stderr += chunk; + }); + + await waitForRpc(child, rpcUrl, () => stderr); + + return { + rpcUrl, + async stop() { + if (child.exitCode !== null) { + return; + } + + child.kill("SIGTERM"); + const close = once(child, "close"); + const timeout = delay(2_000).then(() => { + if (child.exitCode === null) { + child.kill("SIGKILL"); + } + }); + await Promise.race([close, timeout]); + }, + }; +} + +async function runCast( + args: string[], + options: CastOptions = {}, +): Promise { + const executable = + options.executable ?? process.env.ETHDEBUG_CONFORMANCE_CAST ?? "cast"; + + return await new Promise((resolve, reject) => { + const child = spawn(executable, args, { + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (exitCode) => { + if (exitCode !== 0) { + reject( + new Error( + `cast ${args.join(" ")} failed with exit code ${exitCode}\n${stderr}`, + ), + ); + return; + } + + try { + resolve(JSON.parse(stdout) as TransactionReceipt); + } catch (error) { + reject( + new Error( + `cast output was not valid JSON: ${ + error instanceof Error ? error.message : String(error) + }\n${stdout}`, + ), + ); + } + }); + }); +} + +export async function deployBytecode( + anvil: AnvilInstance, + bytecode: string, + options: CastOptions = {}, +): Promise { + const receipt = await runCast( + [ + "send", + "--rpc-url", + anvil.rpcUrl, + "--private-key", + options.privateKey ?? DEFAULT_PRIVATE_KEY, + "--create", + bytecode, + "--json", + ], + options, + ); + + if (!receipt.transactionHash || !receipt.contractAddress) { + throw new Error(`cast deploy did not return a contract address`); + } + return receipt; +} + +export async function sendContractTransaction( + anvil: AnvilInstance, + contractAddress: string, + signature: string, + args: string[], + options: CastOptions = {}, +): Promise { + const receipt = await runCast( + [ + "send", + contractAddress, + signature, + ...args, + "--rpc-url", + anvil.rpcUrl, + "--private-key", + options.privateKey ?? DEFAULT_PRIVATE_KEY, + "--json", + ], + options, + ); + + if (!receipt.transactionHash) { + throw new Error(`cast send did not return a transaction hash`); + } + return receipt; +} diff --git a/packages/conformance/src/adapters/bugc.ts b/packages/conformance/src/adapters/bugc.ts new file mode 100644 index 000000000..d28b511f4 --- /dev/null +++ b/packages/conformance/src/adapters/bugc.ts @@ -0,0 +1,122 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import type { Materials } from "@ethdebug/format"; +import { VERSION, compile } from "@ethdebug/bugc"; + +import type { BugcCompileOptions, EthdebugArtifact } from "../types.js"; + +function hex(bytes: Uint8Array | number[]): string { + return `0x${Buffer.from(bytes).toString("hex")}`; +} + +function relativeSourcePath(sourcePath: string): string { + const relative = path.relative(process.cwd(), sourcePath); + return relative.startsWith("..") ? sourcePath : relative; +} + +function referencedSourceIds(value: unknown): Materials.Id[] { + const ids: Materials.Id[] = []; + function visit(node: unknown): void { + if (!node || typeof node !== "object") { + return; + } + + if ( + "source" in node && + typeof node.source === "object" && + node.source && + "id" in node.source && + ["number", "string"].includes(typeof node.source.id) + ) { + ids.push(node.source.id as Materials.Id); + } + + for (const child of Object.values(node)) { + if (Array.isArray(child)) { + child.forEach(visit); + } else if (child && typeof child === "object") { + visit(child); + } + } + } + + visit(value); + return ids; +} + +export async function compileBugc( + options: BugcCompileOptions, +): Promise { + const sourcePath = path.resolve(options.sourcePath); + const contents = await readFile(sourcePath, "utf8"); + const sourceId = relativeSourcePath(sourcePath); + const result = await compile({ + to: "bytecode", + source: contents, + sourcePath: sourceId, + optimizer: { + level: options.optimizationLevel ?? 0, + }, + }); + + if (!result.success) { + throw new Error( + `BUG compilation failed: ${JSON.stringify(result.messages)}`, + ); + } + + const { bytecode } = result.value; + const programs = [ + { + name: "runtime", + program: bytecode.runtimeProgram, + bytecode: hex(bytecode.runtime), + }, + ]; + + if (bytecode.createProgram && bytecode.create) { + programs.push({ + name: "create", + program: bytecode.createProgram, + bytecode: hex(bytecode.create), + }); + } + + const sourceIds = new Set([sourceId]); + for (const program of programs) { + for (const id of referencedSourceIds(program.program)) { + sourceIds.add(id); + } + } + + const sources = Array.from(sourceIds).map( + (id): Materials.Source => ({ + id, + path: sourceId, + contents, + language: "BUG", + }), + ); + const compilation: Materials.Compilation = { + id: `bugc:${sourceId}`, + compiler: { + name: "bugc", + version: VERSION, + }, + sources, + }; + + return { + compiler: "bugc", + sources, + programs, + compilation, + resources: { + compilation, + types: {}, + pointers: {}, + }, + raw: result.value, + }; +} diff --git a/packages/conformance/src/adapters/solc.ts b/packages/conformance/src/adapters/solc.ts new file mode 100644 index 000000000..f1cfb5b0e --- /dev/null +++ b/packages/conformance/src/adapters/solc.ts @@ -0,0 +1,150 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { spawn } from "node:child_process"; + +import type { + EthdebugArtifact, + EthdebugProgramArtifact, + SourceFile, + SolcCompileOptions, +} from "../types.js"; + +function run( + command: string, + args: string[], + input: string, +): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + stdio: ["pipe", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject( + new Error( + `${command} ${args.join(" ")} failed with exit code ${code}\n${stderr}`, + ), + ); + } + }); + child.stdin.end(input); + }); +} + +function contractOutputs(output: any): EthdebugProgramArtifact[] { + const programs: EthdebugProgramArtifact[] = []; + for (const [sourcePath, contracts] of Object.entries( + output.contracts ?? {}, + )) { + for (const [contractName, contract] of Object.entries(contracts)) { + const createProgram = contract.evm?.bytecode?.ethdebug; + if (createProgram) { + programs.push({ + name: `${sourcePath}:${contractName}:create`, + program: createProgram, + bytecode: contract.evm?.bytecode?.object + ? `0x${contract.evm.bytecode.object}` + : undefined, + }); + } + + const runtimeProgram = contract.evm?.deployedBytecode?.ethdebug; + if (runtimeProgram) { + programs.push({ + name: `${sourcePath}:${contractName}:runtime`, + program: runtimeProgram, + bytecode: contract.evm?.deployedBytecode?.object + ? `0x${contract.evm.deployedBytecode.object}` + : undefined, + }); + } + } + } + return programs; +} + +export async function compileSolc( + options: SolcCompileOptions, +): Promise { + const sourcePaths = options.sourcePaths ?? [options.sourcePath]; + const sourceFiles = await Promise.all( + sourcePaths.map(async (sourcePath): Promise => { + const resolved = path.resolve(sourcePath); + return { + path: path.basename(resolved), + contents: await readFile(resolved, "utf8"), + language: "Solidity", + }; + }), + ); + const solcPath = + options.solcPath ?? process.env.ETHDEBUG_CONFORMANCE_SOLC ?? "solc"; + const input = { + language: "Solidity", + sources: Object.fromEntries( + sourceFiles.map((source) => [ + source.path, + { + content: source.contents, + }, + ]), + ), + settings: { + experimental: true, + viaIR: options.viaIR ?? true, + debug: { + debugInfo: ["ethdebug"], + }, + outputSelection: { + "*": { + "*": [ + "abi", + "evm.bytecode.object", + "evm.bytecode.ethdebug", + "evm.deployedBytecode.object", + "evm.deployedBytecode.ethdebug", + "ethdebug.resources", + "ethdebug.compilation", + ], + }, + }, + }, + }; + + const { stdout } = await run( + solcPath, + ["--standard-json"], + JSON.stringify(input), + ); + const output = JSON.parse(stdout); + + const errors = (output.errors ?? []).filter( + (error: any) => error.severity === "error", + ); + if (errors.length > 0) { + throw new Error(`solc compilation failed: ${JSON.stringify(errors)}`); + } + + return { + compiler: "solc", + sources: sourceFiles, + programs: contractOutputs(output), + compilation: output.ethdebug?.compilation, + resources: output.ethdebug?.resources, + raw: output, + }; +} diff --git a/packages/conformance/src/adapters/soldb.ts b/packages/conformance/src/adapters/soldb.ts new file mode 100644 index 000000000..7e2cada48 --- /dev/null +++ b/packages/conformance/src/adapters/soldb.ts @@ -0,0 +1,128 @@ +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import type { + EthdebugArtifact, + SoldbCommand, + SoldbDebugDir, + SoldbDebugDirOptions, + SoldbResult, +} from "../types.js"; + +export async function runSoldb(command: SoldbCommand): Promise { + const executable = + command.executable ?? process.env.ETHDEBUG_CONFORMANCE_SOLDB ?? "soldb"; + const args = command.args; + + return await new Promise((resolve, reject) => { + const child = spawn(executable, args, { + cwd: command.cwd, + env: { + ...process.env, + ...command.env, + }, + stdio: ["pipe", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (exitCode) => { + let json: unknown; + if (command.expectJson) { + try { + json = JSON.parse(stdout); + } catch (error) { + reject( + new Error( + `SolDB output was not valid JSON: ${ + error instanceof Error ? error.message : String(error) + }\n${stdout}`, + ), + ); + return; + } + } + + resolve({ + command: [executable, ...args], + exitCode, + stdout, + stderr, + json, + }); + }); + + child.stdin.end(command.stdin ?? ""); + }); +} + +function contractName(artifact: EthdebugArtifact, fallback?: string): string { + return ( + fallback ?? + artifact.programs.find((program) => program.program.contract.name)?.program + .contract.name ?? + "Contract" + ); +} + +function ethdebugResources(artifact: EthdebugArtifact): unknown { + if (artifact.resources) { + return artifact.resources; + } + + if (artifact.compilation) { + return { + compilation: artifact.compilation, + types: {}, + pointers: {}, + }; + } + + throw new Error( + "Cannot write SolDB debug directory without ETHDebug resources", + ); +} + +export async function writeSoldbDebugDir( + artifact: EthdebugArtifact, + options: SoldbDebugDirOptions = {}, +): Promise { + const debugDir = + options.dir ?? (await mkdtemp(path.join(os.tmpdir(), "ethdebug-soldb-"))); + await mkdir(debugDir, { recursive: true }); + + const name = contractName(artifact, options.contractName); + const address = + options.address ?? "0x0000000000000000000000000000000000000001"; + + await writeFile( + path.join(debugDir, "ethdebug.json"), + JSON.stringify(ethdebugResources(artifact), null, 2), + ); + + for (const program of artifact.programs) { + const file = + program.program.environment === "create" + ? `${name}_ethdebug.json` + : `${name}_ethdebug-runtime.json`; + await writeFile(path.join(debugDir, file), JSON.stringify(program.program)); + } + + return { + debugDir, + spec: `${address}:${name}:${debugDir}`, + address, + contractName: name, + }; +} diff --git a/packages/conformance/src/index.ts b/packages/conformance/src/index.ts new file mode 100644 index 000000000..b2269430a --- /dev/null +++ b/packages/conformance/src/index.ts @@ -0,0 +1,6 @@ +export * from "./adapters/anvil.js"; +export * from "./adapters/bugc.js"; +export * from "./adapters/soldb.js"; +export * from "./adapters/solc.js"; +export * from "./runner.js"; +export * from "./types.js"; diff --git a/packages/conformance/src/runner.ts b/packages/conformance/src/runner.ts new file mode 100644 index 000000000..8aeac11e1 --- /dev/null +++ b/packages/conformance/src/runner.ts @@ -0,0 +1,147 @@ +import { Materials, isProgram } from "@ethdebug/format"; + +import { compileBugc } from "./adapters/bugc.js"; +import { runSoldb } from "./adapters/soldb.js"; +import { compileSolc } from "./adapters/solc.js"; +import type { + CompileOptions, + ConformanceFixture, + EthdebugArtifact, + SoldbResult, + StaticConformanceIssue, + StaticConformanceResult, +} from "./types.js"; + +function issue(path: string, message: string): StaticConformanceIssue { + return { path, message }; +} + +function sourceIds(artifact: EthdebugArtifact): Set { + const ids = new Set(); + for (const source of artifact.compilation?.sources ?? []) { + ids.add(source.id); + } + for (const source of artifact.resources?.compilation.sources ?? []) { + ids.add(source.id); + } + return ids; +} + +function referencedSourceIds(value: unknown): Materials.Id[] { + const ids: Materials.Id[] = []; + + function visit(node: unknown): void { + if (!node || typeof node !== "object") { + return; + } + + if ( + "source" in node && + typeof node.source === "object" && + node.source && + "id" in node.source && + Materials.isId(node.source.id) + ) { + ids.push(node.source.id); + } + + for (const child of Object.values(node)) { + if (Array.isArray(child)) { + child.forEach(visit); + } else if (child && typeof child === "object") { + visit(child); + } + } + } + + visit(value); + return ids; +} + +export async function compileEthdebug( + options: CompileOptions, +): Promise { + switch (options.kind) { + case "bugc": + return await compileBugc(options); + case "solc": + return await compileSolc(options); + } +} + +export function validateStaticConformance( + artifact: EthdebugArtifact, +): StaticConformanceResult { + const issues: StaticConformanceIssue[] = []; + + if (artifact.programs.length === 0) { + issues.push( + issue("programs", "compiler did not emit any ETHDebug programs"), + ); + } + + artifact.programs.forEach((program, index) => { + if (!isProgram(program.program)) { + issues.push( + issue(`programs[${index}]`, `${program.name} is not a valid program`), + ); + } + }); + + if (artifact.compilation && !Materials.isCompilation(artifact.compilation)) { + issues.push( + issue("compilation", "compilation is not valid materials/compilation"), + ); + } + + if ( + artifact.resources && + !Materials.isCompilation(artifact.resources.compilation) + ) { + issues.push( + issue( + "resources.compilation", + "resources.compilation is not valid materials/compilation", + ), + ); + } + + const knownSourceIds = sourceIds(artifact); + if (knownSourceIds.size > 0) { + artifact.programs.forEach((program, programIndex) => { + referencedSourceIds(program.program).forEach((sourceId) => { + if (!knownSourceIds.has(sourceId)) { + issues.push( + issue( + `programs[${programIndex}]`, + `${program.name} references unknown source id ${String(sourceId)}`, + ), + ); + } + }); + }); + } + + return { + ok: issues.length === 0, + issues, + }; +} + +export async function runConformanceFixture( + fixture: ConformanceFixture, +): Promise<{ + artifact: EthdebugArtifact; + static: StaticConformanceResult; + soldb?: SoldbResult; +}> { + const artifact = await compileEthdebug(fixture.compile); + const staticResult = validateStaticConformance(artifact); + const soldb = fixture.soldb ? await runSoldb(fixture.soldb) : undefined; + + return { + artifact, + static: staticResult, + soldb, + }; +} diff --git a/packages/conformance/src/types.ts b/packages/conformance/src/types.ts new file mode 100644 index 000000000..14910bb65 --- /dev/null +++ b/packages/conformance/src/types.ts @@ -0,0 +1,90 @@ +import type { Materials, Program } from "@ethdebug/format"; + +export type CompilerKind = "bugc" | "solc"; + +export interface SourceFile { + path: string; + contents: string; + language?: string; +} + +export interface EthdebugProgramArtifact { + name: string; + program: Program; + bytecode?: string; +} + +export interface EthdebugArtifact { + compiler: CompilerKind; + sources: SourceFile[]; + programs: EthdebugProgramArtifact[]; + compilation?: Materials.Compilation; + resources?: { + compilation: Materials.Compilation; + types: Record; + pointers: Record; + }; + raw?: unknown; +} + +export interface BugcCompileOptions { + kind: "bugc"; + sourcePath: string; + optimizationLevel?: 0 | 1 | 2 | 3; +} + +export interface SolcCompileOptions { + kind: "solc"; + sourcePath: string; + sourcePaths?: string[]; + solcPath?: string; + viaIR?: boolean; +} + +export type CompileOptions = BugcCompileOptions | SolcCompileOptions; + +export interface SoldbCommand { + executable?: string; + args: string[]; + cwd?: string; + stdin?: string; + env?: Record; + expectJson?: boolean; +} + +export interface SoldbDebugDirOptions { + dir?: string; + address?: string; + contractName?: string; +} + +export interface SoldbDebugDir { + debugDir: string; + spec: string; + address: string; + contractName: string; +} + +export interface SoldbResult { + command: string[]; + exitCode: number | null; + stdout: string; + stderr: string; + json?: unknown; +} + +export interface StaticConformanceIssue { + path: string; + message: string; +} + +export interface StaticConformanceResult { + ok: boolean; + issues: StaticConformanceIssue[]; +} + +export interface ConformanceFixture { + name: string; + compile: CompileOptions; + soldb?: SoldbCommand; +} diff --git a/packages/conformance/test/conformance.test.ts b/packages/conformance/test/conformance.test.ts new file mode 100644 index 000000000..e0e398680 --- /dev/null +++ b/packages/conformance/test/conformance.test.ts @@ -0,0 +1,201 @@ +import { spawnSync } from "node:child_process"; +import { accessSync, constants } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { + deployBytecode, + sendContractTransaction, + startAnvil, +} from "../src/adapters/anvil.js"; +import { runSoldb, writeSoldbDebugDir } from "../src/adapters/soldb.js"; +import { compileEthdebug, validateStaticConformance } from "../src/runner.js"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const solc = process.env.ETHDEBUG_CONFORMANCE_SOLC; +const soldb = process.env.ETHDEBUG_CONFORMANCE_SOLDB; +const anvilExecutable = process.env.ETHDEBUG_CONFORMANCE_ANVIL ?? "anvil"; +const castExecutable = process.env.ETHDEBUG_CONFORMANCE_CAST ?? "cast"; + +function executableExists(command: string): boolean { + if (command.includes("/")) { + try { + accessSync(command, constants.X_OK); + return true; + } catch { + return false; + } + } + + return spawnSync("which", [command], { stdio: "ignore" }).status === 0; +} + +function hasFoundry(): boolean { + return executableExists(anvilExecutable) && executableExists(castExecutable); +} + +describe("@ethdebug/conformance", () => { + it("[bug] compiles BUG fixtures into valid ETHDebug programs", async () => { + const artifact = await compileEthdebug({ + kind: "bugc", + sourcePath: path.join(root, "test/fixtures/bugc/minimal.bug"), + }); + + const result = validateStaticConformance(artifact); + expect(result.issues).toEqual([]); + expect(result.ok).toBe(true); + expect(artifact.programs.length).toBeGreaterThan(0); + }); + + it.skipIf(!solc)( + "[solc] compiles Solidity fixtures and validates ETHDebug output", + async () => { + const artifact = await compileEthdebug({ + kind: "solc", + solcPath: solc!, + sourcePath: path.join(root, "test/fixtures/solc/Counter.sol"), + }); + + const result = validateStaticConformance(artifact); + expect(result.issues).toEqual([]); + expect(result.ok).toBe(true); + expect( + artifact.compilation ?? artifact.resources?.compilation, + ).toBeDefined(); + }, + ); + + it.skipIf(!soldb || !solc)( + "[soldb] checks solc ETHDebug resources through the SolDB consumer backend", + async () => { + const artifact = await compileEthdebug({ + kind: "solc", + solcPath: solc!, + sourcePath: path.join(root, "test/fixtures/solc/Counter.sol"), + }); + const debugDir = await writeSoldbDebugDir(artifact, { + contractName: "Counter", + }); + expect(executableExists(soldb!)).toBe(true); + + const result = await runSoldb({ + executable: soldb!, + args: ["info", "resources", "--ethdebug-dir", debugDir.spec, "--json"], + expectJson: true, + }); + + expect(result.exitCode).toBe(0); + const json = result.json as any; + expect(json.contracts[0].name).toBe("Counter"); + expect(json.contracts[0].resources).toEqual(artifact.resources); + expect( + json.contracts[0].resources.compilation.sources[0].contents, + ).toContain("contract Counter"); + }, + ); + + it.skipIf(!soldb || !solc)( + "[soldb] checks multi-source solc ETHDebug resources through the SolDB consumer backend", + async () => { + const sourceDir = path.join(root, "test/fixtures/solc/multi-source"); + const artifact = await compileEthdebug({ + kind: "solc", + solcPath: solc!, + sourcePath: path.join(sourceDir, "Counter.sol"), + sourcePaths: [ + path.join(sourceDir, "Counter.sol"), + path.join(sourceDir, "Math.sol"), + ], + }); + const result = validateStaticConformance(artifact); + expect(result.issues).toEqual([]); + expect(result.ok).toBe(true); + + const debugDir = await writeSoldbDebugDir(artifact, { + contractName: "Counter", + }); + const soldbResult = await runSoldb({ + executable: soldb!, + args: ["info", "resources", "--ethdebug-dir", debugDir.spec, "--json"], + expectJson: true, + }); + + expect(soldbResult.exitCode).toBe(0); + const json = soldbResult.json as any; + const sources = json.contracts[0].resources.compilation.sources; + expect(sources.map((source: any) => source.path).sort()).toEqual([ + "Counter.sol", + "Math.sol", + ]); + expect( + sources.find((source: any) => source.path === "Counter.sol").contents, + ).toContain('import "./Math.sol"'); + expect( + sources.find((source: any) => source.path === "Math.sol").contents, + ).toContain("library Math"); + }, + ); + + it.skipIf(!soldb || !solc || !hasFoundry())( + "[soldb] checks source-line breakpoints against solc output on local anvil", + async () => { + const artifact = await compileEthdebug({ + kind: "solc", + solcPath: solc!, + sourcePath: path.join(root, "test/fixtures/solc/Counter.sol"), + }); + const createProgram = artifact.programs.find( + (program) => + program.program.environment === "create" && program.bytecode, + ); + expect(createProgram?.bytecode).toBeDefined(); + + const anvil = await startAnvil({ executable: anvilExecutable }); + try { + const deployment = await deployBytecode( + anvil, + createProgram!.bytecode!, + { + executable: castExecutable, + }, + ); + const debugDir = await writeSoldbDebugDir(artifact, { + address: deployment.contractAddress!, + contractName: "Counter", + }); + const tx = await sendContractTransaction( + anvil, + deployment.contractAddress!, + "increment(uint256)", + ["4"], + { executable: castExecutable }, + ); + + const interactive = await runSoldb({ + executable: soldb!, + args: [ + "trace", + tx.transactionHash, + "--rpc", + anvil.rpcUrl, + "--ethdebug-dir", + debugDir.spec, + "--interactive", + ], + stdin: "break Counter.sol:8\ncontinue\nq\n", + }); + + expect(interactive.exitCode).toBe(0); + expect(interactive.stdout).toContain( + "Breakpoint set at Counter.sol:8, PC", + ); + expect(interactive.stdout).toContain("Breakpoint hit at step"); + expect(interactive.stdout).toContain("Counter.sol:8, PC"); + } finally { + await anvil.stop(); + } + }, + ); +}); diff --git a/packages/conformance/test/fixtures/bugc/minimal.bug b/packages/conformance/test/fixtures/bugc/minimal.bug new file mode 100644 index 000000000..b0a08f2b2 --- /dev/null +++ b/packages/conformance/test/fixtures/bugc/minimal.bug @@ -0,0 +1,12 @@ +name MinimalConformance; + +storage { + [0] value: uint256; +} + +create { + value = 1; +} + +code { +} diff --git a/packages/conformance/test/fixtures/solc/Counter.sol b/packages/conformance/test/fixtures/solc/Counter.sol new file mode 100644 index 000000000..fe8810b1e --- /dev/null +++ b/packages/conformance/test/fixtures/solc/Counter.sol @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.29; + +contract Counter { + uint256 public value; + + function increment(uint256 amount) public { + value += amount; + } +} diff --git a/packages/conformance/test/fixtures/solc/multi-source/Counter.sol b/packages/conformance/test/fixtures/solc/multi-source/Counter.sol new file mode 100644 index 000000000..b4a939307 --- /dev/null +++ b/packages/conformance/test/fixtures/solc/multi-source/Counter.sol @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.29; + +import "./Math.sol"; + +contract Counter { + uint256 public value; + + function increment(uint256 amount) public { + value = Math.add(value, amount); + } +} diff --git a/packages/conformance/test/fixtures/solc/multi-source/Math.sol b/packages/conformance/test/fixtures/solc/multi-source/Math.sol new file mode 100644 index 000000000..98a4d2116 --- /dev/null +++ b/packages/conformance/test/fixtures/solc/multi-source/Math.sol @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.29; + +library Math { + function add(uint256 left, uint256 right) internal pure returns (uint256) { + return left + right; + } +} diff --git a/packages/conformance/tsconfig.json b/packages/conformance/tsconfig.json new file mode 100644 index 000000000..2a9d65da9 --- /dev/null +++ b/packages/conformance/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "./", + "outDir": "./dist/", + "baseUrl": "./", + "paths": { + "#adapters/anvil": ["./src/adapters/anvil"], + "#adapters/bugc": ["./src/adapters/bugc"], + "#adapters/soldb": ["./src/adapters/soldb"], + "#adapters/solc": ["./src/adapters/solc"], + "#runner": ["./src/runner"], + "#types": ["./src/types"] + } + }, + "include": ["src/**/*", "test/**/*", "*.ts"], + "exclude": ["node_modules", "dist"], + "references": [{ "path": "../bugc" }, { "path": "../format" }] +} diff --git a/packages/conformance/vitest.config.ts b/packages/conformance/vitest.config.ts new file mode 100644 index 000000000..e78501e06 --- /dev/null +++ b/packages/conformance/vitest.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; +export default defineConfig({ + test: { + environment: "node", + }, +}); From 899f22cfaf114e0a0c1cc54786b360cd645a4b9f Mon Sep 17 00:00:00 2001 From: djole Date: Fri, 15 May 2026 16:47:57 +0200 Subject: [PATCH 2/3] docs: Improve conformance/README.md --- packages/conformance/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/conformance/README.md b/packages/conformance/README.md index 8aeeef8b6..5358dca70 100644 --- a/packages/conformance/README.md +++ b/packages/conformance/README.md @@ -18,14 +18,14 @@ The first layer checks the contract that every compiler should satisfy: - resources and compilations are valid when present, - source references used by programs resolve to compilation sources. -The second layer is Dexter-like consumer conformance: tests can run a debugger -consumer, parse its output, and assert resources, source steps, frames, or -values. SolDB is the first consumer backend, but the runner is not tied to -SolDB. The SolDB adapter can materialize compiler output as a SolDB-compatible -debug directory and then drive `soldb info resources` over it. The optional -Foundry adapter starts a local `anvil --steps-tracing` node, deploys a compiled -contract with `cast`, sends a transaction, and scripts SolDB's interactive REPL -to assert source-line breakpoint set/hit behavior. +The second layer checks real debugger consumers: tests can run a debugger, +parse its output, and assert resources, source steps, frames, or values. SolDB +is the first consumer backend, but the runner is not tied to SolDB. The SolDB +adapter can materialize compiler output as a SolDB-compatible debug directory +and then drive `soldb info resources` over it. The optional Foundry adapter +starts a local `anvil --steps-tracing` node, deploys a compiled contract with +`cast`, sends a transaction, and scripts SolDB's interactive REPL to assert +source-line breakpoint set/hit behavior. External adapters are opt-in in tests: From e61b81f63fe1f7793d2f6ae694ecda02a706a30c Mon Sep 17 00:00:00 2001 From: djole Date: Tue, 9 Jun 2026 14:57:52 +0200 Subject: [PATCH 3/3] fix: address comments --- .github/workflows/ethdebug.yml | 49 +++- packages/conformance/README.md | 10 + packages/conformance/package.json | 3 +- packages/conformance/src/adapters/soldb.ts | 15 ++ packages/conformance/src/runner.ts | 109 +++++++-- packages/conformance/test/conformance.test.ts | 229 +++++++++++++++++- 6 files changed, 377 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ethdebug.yml b/.github/workflows/ethdebug.yml index c9d02af34..c90884c1d 100644 --- a/.github/workflows/ethdebug.yml +++ b/.github/workflows/ethdebug.yml @@ -2,11 +2,19 @@ name: ETHDebug on: pull_request: + paths: + - ".github/workflows/ethdebug.yml" + - "package.json" + - "yarn.lock" + - "tsconfig*.json" + - "schemas/**" + - "packages/format/**" + - "packages/conformance/**" workflow_dispatch: inputs: solidity_ref: description: Solidity ref to build for solc - default: feature/ethdebug + default: develop required: true soldb_ref: description: SolDB ref to build @@ -19,7 +27,7 @@ concurrency: env: FOUNDRY_VERSION: v1.0.0 - SOLIDITY_REF: ${{ github.event.inputs.solidity_ref || 'feature/ethdebug' }} + SOLIDITY_REF: ${{ github.event.inputs.solidity_ref || 'develop' }} SOLDB_REF: ${{ github.event.inputs.soldb_ref || 'main' }} jobs: @@ -33,14 +41,6 @@ jobs: with: path: format - - name: Checkout Solidity - uses: actions/checkout@v4 - with: - repository: walnuthq/solidity - ref: ${{ env.SOLIDITY_REF }} - path: solidity - submodules: recursive - - name: Checkout SolDB uses: actions/checkout@v4 with: @@ -59,6 +59,23 @@ jobs: run: yarn install --frozen-lockfile working-directory: format + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Fetch prebuilt solc with ETHDebug support + id: fetch-solc + continue-on-error: true + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + run: | + python -m pip install git+https://github.com/argotorg/solc-bench.git + solc-bench fetch "${SOLIDITY_REF}" \ + --output "${GITHUB_WORKSPACE}/solidity/build/solc/solc" \ + --force + - name: Install Rust toolchain run: | rustup toolchain install stable --profile minimal @@ -79,7 +96,17 @@ jobs: run: cargo build --bin soldb working-directory: soldb + - name: Checkout Solidity + if: steps.fetch-solc.outcome != 'success' + uses: actions/checkout@v4 + with: + repository: argotorg/solidity + ref: ${{ env.SOLIDITY_REF }} + path: solidity + submodules: recursive + - name: Install Solidity build dependencies + if: steps.fetch-solc.outcome != 'success' run: | sudo apt-get update sudo apt-get install -y \ @@ -94,6 +121,7 @@ jobs: ninja-build - name: Cache Solidity ccache + if: steps.fetch-solc.outcome != 'success' uses: actions/cache@v4 with: path: ~/.ccache @@ -102,6 +130,7 @@ jobs: ${{ runner.os }}-solidity-ccache- - name: Build solc with ETHDebug support + if: steps.fetch-solc.outcome != 'success' run: | cmake \ -S solidity \ diff --git a/packages/conformance/README.md b/packages/conformance/README.md index 5358dca70..8ebb94b99 100644 --- a/packages/conformance/README.md +++ b/packages/conformance/README.md @@ -18,6 +18,11 @@ The first layer checks the contract that every compiler should satisfy: - resources and compilations are valid when present, - source references used by programs resolve to compilation sources. +Static validation uses both the package's TypeScript type guards and the +published JSON-Schemas. The schemas are normative; the type guards remain as an +extra compatibility signal for TypeScript consumers. Negative fixtures are kept +alongside positive fixtures to prove the validator rejects malformed artifacts. + The second layer checks real debugger consumers: tests can run a debugger, parse its output, and assert resources, source steps, frames, or values. SolDB is the first consumer backend, but the runner is not tied to SolDB. The SolDB @@ -27,6 +32,11 @@ starts a local `anvil --steps-tracing` node, deploys a compiled contract with `cast`, sends a transaction, and scripts SolDB's interactive REPL to assert source-line breakpoint set/hit behavior. +Consumer adapters are black-box by default so the same harness can test CLIs, +libraries, or services. The current SolDB adapter shells out to the CLI because +that is the stable public integration point; a future SolDB library adapter can +be added without changing the compiler conformance layer. + External adapters are opt-in in tests: ```console diff --git a/packages/conformance/package.json b/packages/conformance/package.json index 2812f3ff6..586ee8e3f 100644 --- a/packages/conformance/package.json +++ b/packages/conformance/package.json @@ -44,7 +44,8 @@ }, "dependencies": { "@ethdebug/bugc": "^0.1.0-0", - "@ethdebug/format": "^0.1.0-0" + "@ethdebug/format": "^0.1.0-0", + "@hyperjump/json-schema": "^1.17.3" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/packages/conformance/src/adapters/soldb.ts b/packages/conformance/src/adapters/soldb.ts index 7e2cada48..bf319791d 100644 --- a/packages/conformance/src/adapters/soldb.ts +++ b/packages/conformance/src/adapters/soldb.ts @@ -67,6 +67,21 @@ export async function runSoldb(command: SoldbCommand): Promise { }); } +export function sourceBreakpointScript(target: string): string { + return `break ${target}\ncontinue\nq\n`; +} + +export function observeSourceBreakpoint( + result: SoldbResult, + target: string, +): { set: boolean; hit: boolean; stoppedAtTarget: boolean } { + return { + set: result.stdout.includes(`Breakpoint set at ${target}, PC`), + hit: result.stdout.includes("Breakpoint hit at step"), + stoppedAtTarget: result.stdout.includes(`${target}, PC`), + }; +} + function contractName(artifact: EthdebugArtifact, fallback?: string): string { return ( fallback ?? diff --git a/packages/conformance/src/runner.ts b/packages/conformance/src/runner.ts index 8aeac11e1..64d5bfcaf 100644 --- a/packages/conformance/src/runner.ts +++ b/packages/conformance/src/runner.ts @@ -1,4 +1,11 @@ -import { Materials, isProgram } from "@ethdebug/format"; +import { Materials, isProgram, schemas } from "@ethdebug/format"; +import { + addSchema, + setMetaSchemaOutputFormat, + validate, + type OutputUnit, +} from "@hyperjump/json-schema/draft-2020-12"; +import { BASIC } from "@hyperjump/json-schema/experimental"; import { compileBugc } from "./adapters/bugc.js"; import { runSoldb } from "./adapters/soldb.js"; @@ -16,6 +23,52 @@ function issue(path: string, message: string): StaticConformanceIssue { return { path, message }; } +let schemasRegistered = false; + +function registerSchemas(): void { + if (schemasRegistered) { + return; + } + + setMetaSchemaOutputFormat(BASIC); + for (const schema of Object.values(schemas)) { + addSchema(schema as any); + } + schemasRegistered = true; +} + +function schemaErrors(output: { errors?: OutputUnit[] }): string { + const errors = output.errors ?? []; + const messages = errors + .map((error) => { + if (error.valid || error.keyword.endsWith("#validate")) { + return undefined; + } + return `${error.instanceLocation} fails ${error.absoluteKeywordLocation}`; + }) + .filter((message): message is string => !!message); + + return messages.length > 0 ? messages.join("; ") : "schema validation failed"; +} + +async function validateSchema( + schemaId: string, + value: unknown, + path: string, + issues: StaticConformanceIssue[], +): Promise { + registerSchemas(); + const output = await validate(schemaId, value as any, BASIC); + if (!output.valid) { + issues.push( + issue( + path, + `does not validate against ${schemaId}: ${schemaErrors(output)}`, + ), + ); + } +} + function sourceIds(artifact: EthdebugArtifact): Set { const ids = new Set(); for (const source of artifact.compilation?.sources ?? []) { @@ -69,9 +122,9 @@ export async function compileEthdebug( } } -export function validateStaticConformance( +export async function validateStaticConformance( artifact: EthdebugArtifact, -): StaticConformanceResult { +): Promise { const issues: StaticConformanceIssue[] = []; if (artifact.programs.length === 0) { @@ -87,12 +140,28 @@ export function validateStaticConformance( ); } }); + for (const [index, program] of artifact.programs.entries()) { + await validateSchema( + "schema:ethdebug/format/program", + program.program, + `programs[${index}]`, + issues, + ); + } if (artifact.compilation && !Materials.isCompilation(artifact.compilation)) { issues.push( issue("compilation", "compilation is not valid materials/compilation"), ); } + if (artifact.compilation) { + await validateSchema( + "schema:ethdebug/format/materials/compilation", + artifact.compilation, + "compilation", + issues, + ); + } if ( artifact.resources && @@ -105,22 +174,28 @@ export function validateStaticConformance( ), ); } + if (artifact.resources) { + await validateSchema( + "schema:ethdebug/format/info/resources", + artifact.resources, + "resources", + issues, + ); + } const knownSourceIds = sourceIds(artifact); - if (knownSourceIds.size > 0) { - artifact.programs.forEach((program, programIndex) => { - referencedSourceIds(program.program).forEach((sourceId) => { - if (!knownSourceIds.has(sourceId)) { - issues.push( - issue( - `programs[${programIndex}]`, - `${program.name} references unknown source id ${String(sourceId)}`, - ), - ); - } - }); + artifact.programs.forEach((program, programIndex) => { + referencedSourceIds(program.program).forEach((sourceId) => { + if (!knownSourceIds.has(sourceId)) { + issues.push( + issue( + `programs[${programIndex}]`, + `${program.name} references unknown source id ${String(sourceId)}`, + ), + ); + } }); - } + }); return { ok: issues.length === 0, @@ -136,7 +211,7 @@ export async function runConformanceFixture( soldb?: SoldbResult; }> { const artifact = await compileEthdebug(fixture.compile); - const staticResult = validateStaticConformance(artifact); + const staticResult = await validateStaticConformance(artifact); const soldb = fixture.soldb ? await runSoldb(fixture.soldb) : undefined; return { diff --git a/packages/conformance/test/conformance.test.ts b/packages/conformance/test/conformance.test.ts index e0e398680..1150cf394 100644 --- a/packages/conformance/test/conformance.test.ts +++ b/packages/conformance/test/conformance.test.ts @@ -1,5 +1,6 @@ import { spawnSync } from "node:child_process"; import { accessSync, constants } from "node:fs"; +import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -10,8 +11,14 @@ import { sendContractTransaction, startAnvil, } from "../src/adapters/anvil.js"; -import { runSoldb, writeSoldbDebugDir } from "../src/adapters/soldb.js"; +import { + observeSourceBreakpoint, + runSoldb, + sourceBreakpointScript, + writeSoldbDebugDir, +} from "../src/adapters/soldb.js"; import { compileEthdebug, validateStaticConformance } from "../src/runner.js"; +import type { EthdebugArtifact } from "../src/types.js"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const solc = process.env.ETHDEBUG_CONFORMANCE_SOLC; @@ -36,6 +43,105 @@ function hasFoundry(): boolean { return executableExists(anvilExecutable) && executableExists(castExecutable); } +function validCompilation() { + return { + id: "test-compilation", + compiler: { + name: "testc", + version: "0.0.0", + }, + sources: [ + { + id: 5, + path: "Counter.test", + contents: "contract Counter {}", + language: "Test", + }, + ], + }; +} + +function validProgram() { + return { + contract: { + name: "Counter", + definition: { + source: { + id: 5, + }, + range: { + offset: 0, + length: 19, + }, + }, + }, + environment: "call", + instructions: [ + { + offset: 0, + operation: { + mnemonic: "STOP", + }, + context: { + code: { + source: { + id: 5, + }, + range: { + offset: 0, + length: 19, + }, + }, + }, + }, + ], + }; +} + +function validResources() { + return { + compilation: validCompilation(), + types: { + CounterSlot: { + kind: "uint", + bits: 256, + }, + }, + pointers: { + CounterSlotStorage: { + expect: ["slot"], + for: { + location: "storage", + slot: "slot", + }, + }, + }, + }; +} + +function validArtifact( + overrides: Partial = {}, +): EthdebugArtifact { + return { + compiler: "solc", + sources: [ + { + path: "Counter.test", + contents: "contract Counter {}", + language: "Test", + }, + ], + programs: [ + { + name: "Counter:runtime", + program: validProgram() as any, + }, + ], + compilation: validCompilation() as any, + ...overrides, + }; +} + describe("@ethdebug/conformance", () => { it("[bug] compiles BUG fixtures into valid ETHDebug programs", async () => { const artifact = await compileEthdebug({ @@ -43,7 +149,7 @@ describe("@ethdebug/conformance", () => { sourcePath: path.join(root, "test/fixtures/bugc/minimal.bug"), }); - const result = validateStaticConformance(artifact); + const result = await validateStaticConformance(artifact); expect(result.issues).toEqual([]); expect(result.ok).toBe(true); expect(artifact.programs.length).toBeGreaterThan(0); @@ -58,7 +164,7 @@ describe("@ethdebug/conformance", () => { sourcePath: path.join(root, "test/fixtures/solc/Counter.sol"), }); - const result = validateStaticConformance(artifact); + const result = await validateStaticConformance(artifact); expect(result.issues).toEqual([]); expect(result.ok).toBe(true); expect( @@ -109,7 +215,7 @@ describe("@ethdebug/conformance", () => { path.join(sourceDir, "Math.sol"), ], }); - const result = validateStaticConformance(artifact); + const result = await validateStaticConformance(artifact); expect(result.issues).toEqual([]); expect(result.ok).toBe(true); @@ -184,18 +290,121 @@ describe("@ethdebug/conformance", () => { debugDir.spec, "--interactive", ], - stdin: "break Counter.sol:8\ncontinue\nq\n", + stdin: sourceBreakpointScript("Counter.sol:8"), }); expect(interactive.exitCode).toBe(0); - expect(interactive.stdout).toContain( - "Breakpoint set at Counter.sol:8, PC", - ); - expect(interactive.stdout).toContain("Breakpoint hit at step"); - expect(interactive.stdout).toContain("Counter.sol:8, PC"); + expect(observeSourceBreakpoint(interactive, "Counter.sol:8")).toEqual({ + set: true, + hit: true, + stoppedAtTarget: true, + }); } finally { await anvil.stop(); } }, ); + + it("rejects malformed ETHDebug programs through JSON-Schema validation", async () => { + const artifact = validArtifact({ + programs: [ + { + name: "Counter:runtime", + program: { + ...validProgram(), + instructions: "not an instruction array", + } as any, + }, + ], + }); + + const result = await validateStaticConformance(artifact); + + expect(result.ok).toBe(false); + expect(result.issues.some((issue) => issue.path === "programs[0]")).toBe( + true, + ); + expect( + result.issues.some((issue) => + issue.message.includes("schema:ethdebug/format/program"), + ), + ).toBe(true); + }); + + it("rejects program source references when no compilation source table is present", async () => { + const result = await validateStaticConformance( + validArtifact({ + compilation: undefined, + resources: undefined, + }), + ); + + expect(result.ok).toBe(false); + expect( + result.issues.some((issue) => + issue.message.includes("references unknown source id 5"), + ), + ).toBe(true); + }); + + it("validates resources lookup tables for types and pointer templates", async () => { + const artifact = validArtifact({ + compilation: undefined, + resources: validResources() as any, + }); + + const result = await validateStaticConformance(artifact); + + expect(result.issues).toEqual([]); + expect(result.ok).toBe(true); + }); + + it("rejects malformed resources lookup tables through JSON-Schema validation", async () => { + const artifact = validArtifact({ + compilation: undefined, + resources: { + ...validResources(), + pointers: { + CounterSlotStorage: { + expect: ["slot"], + }, + }, + } as any, + }); + + const result = await validateStaticConformance(artifact); + + expect(result.ok).toBe(false); + expect(result.issues.some((issue) => issue.path === "resources")).toBe( + true, + ); + }); + + it("materializes non-empty resources into SolDB debug directories", async () => { + const debugDir = await writeSoldbDebugDir( + validArtifact({ + compilation: undefined, + resources: validResources() as any, + }), + { + contractName: "Counter", + }, + ); + + const resources = JSON.parse( + await readFile(path.join(debugDir.debugDir, "ethdebug.json"), "utf8"), + ); + + expect(resources.types.CounterSlot).toEqual({ + kind: "uint", + bits: 256, + }); + expect(resources.pointers.CounterSlotStorage).toEqual({ + expect: ["slot"], + for: { + location: "storage", + slot: "slot", + }, + }); + }); });