From 0d29d51f72bd9df6532e85a41f38b77524c662ab Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 23:30:40 +0000 Subject: [PATCH] =?UTF-8?q?gates:=20diff-test=20orchestration=20=E2=80=94?= =?UTF-8?q?=20corpus=20pin=20+=20tests=20+=20golden=20per=20target?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the diff-test CLI ('just gates '): loads targets//target.toml (new [corpus] section: manifest, root, golden_sha256; commands now repo-root-relative), verifies the corpus manifest FIRST and refuses to run any gate on mismatch, then runs the upstream test suite and golden replay (stdout sha256 vs pinned hash). Differential fuzz and sanitizers remain per-attempt manual steps in Phase 1 and are reported as skipped-with-reason; all-skipped is not a pass. Verified end-to-end on both vendored targets (comrak: 1 pinned file, 848 tests, golden PASS; tokei: 545 pinned files, 250 tests, golden PASS) and the refusal path (tampered corpus -> 'HASH MISMATCH ... refusing to run' before any gate executes). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2 --- CLAUDE.md | 1 + Cargo.lock | 1 + crates/diff-test/Cargo.toml | 1 + crates/diff-test/src/lib.rs | 1 + crates/diff-test/src/main.rs | 93 ++++++++++++++++++++++++++++++++++ crates/diff-test/src/target.rs | 72 ++++++++++++++++++++++++++ justfile | 5 ++ targets/comrak/target.toml | 11 ++-- targets/tokei/target.toml | 11 ++-- 9 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 crates/diff-test/src/main.rs create mode 100644 crates/diff-test/src/target.rs diff --git a/CLAUDE.md b/CLAUDE.md index 86ac6a7..253cb9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,6 +53,7 @@ Built so far (command-level, pre-target-integration): just build / test / lint # trust-layer workspace: cargo build/test/clippy+fmt just aa [cmd] # A/A self-test of bench-runner on a shell command just compare # interleaved A/B of two shell commands, bootstrap CI +just gates # corpus pin + upstream tests + golden replay just pin-check # verify corpus MANIFEST.sha256 just calibrate # automated A/A + regression-injection calibration ``` diff --git a/Cargo.lock b/Cargo.lock index da1e0af..8710e86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,6 +183,7 @@ name = "diff-test" version = "0.1.0" dependencies = [ "anyhow", + "clap", "serde", "sha2", "thiserror", diff --git a/crates/diff-test/Cargo.toml b/crates/diff-test/Cargo.toml index 4c12796..a516f4a 100644 --- a/crates/diff-test/Cargo.toml +++ b/crates/diff-test/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true [dependencies] anyhow.workspace = true serde.workspace = true +clap.workspace = true sha2.workspace = true thiserror.workspace = true toml.workspace = true diff --git a/crates/diff-test/src/lib.rs b/crates/diff-test/src/lib.rs index 07f8e94..0c44a8c 100644 --- a/crates/diff-test/src/lib.rs +++ b/crates/diff-test/src/lib.rs @@ -8,6 +8,7 @@ //! declares an explicit FP tolerance). pub mod pin; +pub mod target; pub mod policy; use serde::{Deserialize, Serialize}; diff --git a/crates/diff-test/src/main.rs b/crates/diff-test/src/main.rs new file mode 100644 index 0000000..678356b --- /dev/null +++ b/crates/diff-test/src/main.rs @@ -0,0 +1,93 @@ +//! Gate orchestration: corpus pin โ†’ upstream tests โ†’ golden replay. +//! Refuses to run anything if the corpus manifest mismatches (SPEC ยง3.2). +//! Differential fuzz and sanitizers remain per-attempt manual steps in +//! Phase 1 and are reported as skipped with a reason. + +use anyhow::Result; +use clap::Parser; +use diff_test::target::{expected_golden_hash, TargetSpec}; +use diff_test::{all_passed, pin, GateLayer, GateOutcome}; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +#[derive(Parser)] +#[command(name = "diff-test", about = "hotpath equivalence-gate orchestrator")] +struct Cli { + /// Target name under targets/. + target: String, + /// Repository root (defaults to current directory). + #[arg(long, default_value = ".")] + root: PathBuf, +} + +fn sh(root: &PathBuf, cmd: &str) -> Result { + Ok(Command::new("sh") + .arg("-c") + .arg(cmd) + .current_dir(root) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output()?) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let root = cli.root.canonicalize()?; + let spec = TargetSpec::load(&root, &cli.target)?; + let mut results: Vec<(GateLayer, GateOutcome)> = Vec::new(); + + // Corpus pin is a precondition, not a gate: mismatch = refuse to run. + let verified = pin::verify_manifest(&root.join(&spec.corpus.manifest), &root.join(&spec.corpus.root))?; + println!("corpus pin: {} files verified against {}", verified, spec.corpus.manifest.display()); + + print!("upstream tests ... "); + let t = sh(&root, &spec.gates.tests)?; + let tests_outcome = if t.status.success() { + println!("PASS"); + GateOutcome::Passed + } else { + println!("FAIL"); + GateOutcome::Failed { + detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(), + } + }; + results.push((GateLayer::UpstreamTests, tests_outcome)); + + print!("golden replay ... "); + let golden_cmd = spec.gates.golden.replace("{binary}", &spec.build.binary); + let g = sh(&root, &golden_cmd)?; + let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?; + let actual = format!("{:x}", Sha256::digest(&g.stdout)); + let golden_outcome = if g.status.success() && actual == expected { + println!("PASS (stdout sha256 = pinned golden)"); + GateOutcome::Passed + } else { + println!("FAIL"); + GateOutcome::Failed { + detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()), + } + }; + results.push((GateLayer::GoldenReplay, golden_outcome)); + + results.push(( + GateLayer::DifferentialFuzz, + GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into() }, + )); + results.push(( + GateLayer::Sanitizers, + GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into() }, + )); + + for (layer, outcome) in &results { + if let GateOutcome::Skipped { reason } = outcome { + println!("{layer:?}: skipped โ€” {reason}"); + } + } + if all_passed(&results) { + println!("gates: PASS ({})", cli.target); + Ok(()) + } else { + anyhow::bail!("gates: FAIL ({})", cli.target) + } +} diff --git a/crates/diff-test/src/target.rs b/crates/diff-test/src/target.rs new file mode 100644 index 0000000..e11b9bc --- /dev/null +++ b/crates/diff-test/src/target.rs @@ -0,0 +1,72 @@ +//! Per-target spec (`targets//target.toml`). All commands and +//! paths are relative to the repository root; `{binary}` in commands is +//! substituted with the built binary path. + +use serde::Deserialize; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Deserialize)] +pub struct TargetSpec { + pub source: Source, + pub build: Build, + pub gates: Gates, + pub corpus: Corpus, +} + +#[derive(Debug, Deserialize)] +pub struct Source { + pub repo: String, + pub commit: String, + pub license: String, + #[serde(default)] + pub submodules: toml::value::Table, +} + +#[derive(Debug, Deserialize)] +pub struct Build { + /// Baseline build command, run from repo root. + pub baseline: String, + /// Built binary path, relative to repo root. + pub binary: String, +} + +#[derive(Debug, Deserialize)] +pub struct Gates { + /// Upstream test suite command, run from repo root. + pub tests: String, + /// Golden replay command; stdout is hashed. `{binary}` substituted. + pub golden: String, +} + +#[derive(Debug, Deserialize)] +pub struct Corpus { + /// sha256sum-format manifest of the corpus inputs. + pub manifest: PathBuf, + /// Directory the manifest's relative paths resolve against. + pub root: PathBuf, + /// File whose last whitespace-delimited-first-field line is the + /// expected sha256 of the golden command's stdout. + pub golden_sha256: PathBuf, +} + +impl TargetSpec { + pub fn load(repo_root: &Path, name: &str) -> anyhow::Result { + let path = repo_root.join("targets").join(name).join("target.toml"); + let raw = std::fs::read_to_string(&path) + .map_err(|e| anyhow::anyhow!("cannot read {}: {e}", path.display()))?; + Ok(toml::from_str(&raw)?) + } +} + +/// Parse the expected stdout hash out of a GOLDEN.sha256 file +/// (comment lines starting with '#' ignored). +pub fn expected_golden_hash(path: &Path) -> anyhow::Result { + let raw = std::fs::read_to_string(path)?; + raw.lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .filter_map(|l| l.split_whitespace().next()) + .next_back() + .map(str::to_string) + .ok_or_else(|| anyhow::anyhow!("no hash entry in {}", path.display())) +} diff --git a/justfile b/justfile index 9453cf5..f0fcc26 100644 --- a/justfile +++ b/justfile @@ -33,3 +33,8 @@ pin-check target: calibrate cmd out sessions="20": cargo run -p bench-runner -- --config config/accept.toml calibrate \ --cmd "{{cmd}}" --sessions {{sessions}} --out "{{out}}" + +# Equivalence gates for a target: corpus pin (refuses on mismatch) -> +# upstream tests -> golden replay. Fuzz/sanitizers stay per-attempt. +gates target: + cargo run -p diff-test -- {{target}} diff --git a/targets/comrak/target.toml b/targets/comrak/target.toml index f1d7945..4fc885d 100644 --- a/targets/comrak/target.toml +++ b/targets/comrak/target.toml @@ -18,8 +18,8 @@ license = "BSD-2-Clause" # git clone workspace/vendor/progit && git -C ... checkout [build] -baseline = "cargo build --release --manifest-path workspace/Cargo.toml" -binary = "workspace/target/release/comrak" +baseline = "cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml" +binary = "targets/comrak/workspace/target/release/comrak" [bench] # Upstream workload (benches/bench.sh): full Pro Git book through the @@ -28,6 +28,11 @@ command = "cat corpora/comrak/progit.md | {binary} > /dev/null" workload = "Pro Git book, all translations, 11.0 MB concatenated markdown, single stream" [gates] -tests = "cargo test --release --manifest-path workspace/Cargo.toml" +tests = "cargo test --release --manifest-path targets/comrak/workspace/Cargo.toml" # Golden replay: corpus through the binary, stdout must be byte-identical. golden = "cat corpora/comrak/progit.md | {binary}" + +[corpus] +manifest = "corpora/comrak/MANIFEST.sha256" +root = "corpora/comrak" +golden_sha256 = "corpora/comrak/GOLDEN.sha256" diff --git a/targets/tokei/target.toml b/targets/tokei/target.toml index 4f32fc6..f7f33b9 100644 --- a/targets/tokei/target.toml +++ b/targets/tokei/target.toml @@ -9,8 +9,8 @@ commit = "fa44e5194060305576514d59b850353643afbfc8" # v14.0.0 license = "MIT OR Apache-2.0" [build] -baseline = "cargo build --release --manifest-path workspace/Cargo.toml" -binary = "workspace/target/release/tokei" +baseline = "cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml" +binary = "targets/tokei/workspace/target/release/tokei" [bench] # Single rayon thread: measurement first (SPEC conventions); the @@ -19,5 +19,10 @@ command = "RAYON_NUM_THREADS=1 {binary} --sort code corpora/tokei/input > /dev/n workload = "8 replicas of pinned progit + comrak/src trees, 221 MB, 4360 files, single rayon thread" [gates] -tests = "cargo test --release --manifest-path workspace/Cargo.toml" +tests = "cargo test --release --manifest-path targets/tokei/workspace/Cargo.toml" golden = "RAYON_NUM_THREADS=1 {binary} --sort code corpora/tokei/input" + +[corpus] +manifest = "corpora/tokei/MANIFEST.sha256" +root = "corpora/tokei/input/copy1" +golden_sha256 = "corpora/tokei/GOLDEN.sha256"