diff --git a/CLAUDE.md b/CLAUDE.md index 86ac6a7..253cb9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,6 +53,7 @@ Built so far (command-level, pre-target-integration): just build / test / lint # trust-layer workspace: cargo build/test/clippy+fmt just aa [cmd] # A/A self-test of bench-runner on a shell command just compare # interleaved A/B of two shell commands, bootstrap CI +just gates # corpus pin + upstream tests + golden replay just pin-check # verify corpus MANIFEST.sha256 just calibrate # automated A/A + regression-injection calibration ``` diff --git a/Cargo.lock b/Cargo.lock index da1e0af..8710e86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,6 +183,7 @@ name = "diff-test" version = "0.1.0" dependencies = [ "anyhow", + "clap", "serde", "sha2", "thiserror", diff --git a/crates/diff-test/Cargo.toml b/crates/diff-test/Cargo.toml index 4c12796..a516f4a 100644 --- a/crates/diff-test/Cargo.toml +++ b/crates/diff-test/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true [dependencies] anyhow.workspace = true serde.workspace = true +clap.workspace = true sha2.workspace = true thiserror.workspace = true toml.workspace = true diff --git a/crates/diff-test/src/lib.rs b/crates/diff-test/src/lib.rs index 07f8e94..0c44a8c 100644 --- a/crates/diff-test/src/lib.rs +++ b/crates/diff-test/src/lib.rs @@ -8,6 +8,7 @@ //! declares an explicit FP tolerance). pub mod pin; +pub mod target; pub mod policy; use serde::{Deserialize, Serialize}; diff --git a/crates/diff-test/src/main.rs b/crates/diff-test/src/main.rs new file mode 100644 index 0000000..678356b --- /dev/null +++ b/crates/diff-test/src/main.rs @@ -0,0 +1,93 @@ +//! Gate orchestration: corpus pin โ†’ upstream tests โ†’ golden replay. +//! Refuses to run anything if the corpus manifest mismatches (SPEC ยง3.2). +//! Differential fuzz and sanitizers remain per-attempt manual steps in +//! Phase 1 and are reported as skipped with a reason. + +use anyhow::Result; +use clap::Parser; +use diff_test::target::{expected_golden_hash, TargetSpec}; +use diff_test::{all_passed, pin, GateLayer, GateOutcome}; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +#[derive(Parser)] +#[command(name = "diff-test", about = "hotpath equivalence-gate orchestrator")] +struct Cli { + /// Target name under targets/. + target: String, + /// Repository root (defaults to current directory). + #[arg(long, default_value = ".")] + root: PathBuf, +} + +fn sh(root: &PathBuf, cmd: &str) -> Result { + Ok(Command::new("sh") + .arg("-c") + .arg(cmd) + .current_dir(root) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output()?) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let root = cli.root.canonicalize()?; + let spec = TargetSpec::load(&root, &cli.target)?; + let mut results: Vec<(GateLayer, GateOutcome)> = Vec::new(); + + // Corpus pin is a precondition, not a gate: mismatch = refuse to run. + let verified = pin::verify_manifest(&root.join(&spec.corpus.manifest), &root.join(&spec.corpus.root))?; + println!("corpus pin: {} files verified against {}", verified, spec.corpus.manifest.display()); + + print!("upstream tests ... "); + let t = sh(&root, &spec.gates.tests)?; + let tests_outcome = if t.status.success() { + println!("PASS"); + GateOutcome::Passed + } else { + println!("FAIL"); + GateOutcome::Failed { + detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(), + } + }; + results.push((GateLayer::UpstreamTests, tests_outcome)); + + print!("golden replay ... "); + let golden_cmd = spec.gates.golden.replace("{binary}", &spec.build.binary); + let g = sh(&root, &golden_cmd)?; + let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?; + let actual = format!("{:x}", Sha256::digest(&g.stdout)); + let golden_outcome = if g.status.success() && actual == expected { + println!("PASS (stdout sha256 = pinned golden)"); + GateOutcome::Passed + } else { + println!("FAIL"); + GateOutcome::Failed { + detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()), + } + }; + results.push((GateLayer::GoldenReplay, golden_outcome)); + + results.push(( + GateLayer::DifferentialFuzz, + GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into() }, + )); + results.push(( + GateLayer::Sanitizers, + GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into() }, + )); + + for (layer, outcome) in &results { + if let GateOutcome::Skipped { reason } = outcome { + println!("{layer:?}: skipped โ€” {reason}"); + } + } + if all_passed(&results) { + println!("gates: PASS ({})", cli.target); + Ok(()) + } else { + anyhow::bail!("gates: FAIL ({})", cli.target) + } +} diff --git a/crates/diff-test/src/target.rs b/crates/diff-test/src/target.rs new file mode 100644 index 0000000..e11b9bc --- /dev/null +++ b/crates/diff-test/src/target.rs @@ -0,0 +1,72 @@ +//! Per-target spec (`targets//target.toml`). All commands and +//! paths are relative to the repository root; `{binary}` in commands is +//! substituted with the built binary path. + +use serde::Deserialize; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Deserialize)] +pub struct TargetSpec { + pub source: Source, + pub build: Build, + pub gates: Gates, + pub corpus: Corpus, +} + +#[derive(Debug, Deserialize)] +pub struct Source { + pub repo: String, + pub commit: String, + pub license: String, + #[serde(default)] + pub submodules: toml::value::Table, +} + +#[derive(Debug, Deserialize)] +pub struct Build { + /// Baseline build command, run from repo root. + pub baseline: String, + /// Built binary path, relative to repo root. + pub binary: String, +} + +#[derive(Debug, Deserialize)] +pub struct Gates { + /// Upstream test suite command, run from repo root. + pub tests: String, + /// Golden replay command; stdout is hashed. `{binary}` substituted. + pub golden: String, +} + +#[derive(Debug, Deserialize)] +pub struct Corpus { + /// sha256sum-format manifest of the corpus inputs. + pub manifest: PathBuf, + /// Directory the manifest's relative paths resolve against. + pub root: PathBuf, + /// File whose last whitespace-delimited-first-field line is the + /// expected sha256 of the golden command's stdout. + pub golden_sha256: PathBuf, +} + +impl TargetSpec { + pub fn load(repo_root: &Path, name: &str) -> anyhow::Result { + let path = repo_root.join("targets").join(name).join("target.toml"); + let raw = std::fs::read_to_string(&path) + .map_err(|e| anyhow::anyhow!("cannot read {}: {e}", path.display()))?; + Ok(toml::from_str(&raw)?) + } +} + +/// Parse the expected stdout hash out of a GOLDEN.sha256 file +/// (comment lines starting with '#' ignored). +pub fn expected_golden_hash(path: &Path) -> anyhow::Result { + let raw = std::fs::read_to_string(path)?; + raw.lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .filter_map(|l| l.split_whitespace().next()) + .next_back() + .map(str::to_string) + .ok_or_else(|| anyhow::anyhow!("no hash entry in {}", path.display())) +} diff --git a/justfile b/justfile index 9453cf5..f0fcc26 100644 --- a/justfile +++ b/justfile @@ -33,3 +33,8 @@ pin-check target: calibrate cmd out sessions="20": cargo run -p bench-runner -- --config config/accept.toml calibrate \ --cmd "{{cmd}}" --sessions {{sessions}} --out "{{out}}" + +# Equivalence gates for a target: corpus pin (refuses on mismatch) -> +# upstream tests -> golden replay. Fuzz/sanitizers stay per-attempt. +gates target: + cargo run -p diff-test -- {{target}} diff --git a/targets/comrak/target.toml b/targets/comrak/target.toml index f1d7945..4fc885d 100644 --- a/targets/comrak/target.toml +++ b/targets/comrak/target.toml @@ -18,8 +18,8 @@ license = "BSD-2-Clause" # git clone workspace/vendor/progit && git -C ... checkout [build] -baseline = "cargo build --release --manifest-path workspace/Cargo.toml" -binary = "workspace/target/release/comrak" +baseline = "cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml" +binary = "targets/comrak/workspace/target/release/comrak" [bench] # Upstream workload (benches/bench.sh): full Pro Git book through the @@ -28,6 +28,11 @@ command = "cat corpora/comrak/progit.md | {binary} > /dev/null" workload = "Pro Git book, all translations, 11.0 MB concatenated markdown, single stream" [gates] -tests = "cargo test --release --manifest-path workspace/Cargo.toml" +tests = "cargo test --release --manifest-path targets/comrak/workspace/Cargo.toml" # Golden replay: corpus through the binary, stdout must be byte-identical. golden = "cat corpora/comrak/progit.md | {binary}" + +[corpus] +manifest = "corpora/comrak/MANIFEST.sha256" +root = "corpora/comrak" +golden_sha256 = "corpora/comrak/GOLDEN.sha256" diff --git a/targets/tokei/target.toml b/targets/tokei/target.toml index 4f32fc6..f7f33b9 100644 --- a/targets/tokei/target.toml +++ b/targets/tokei/target.toml @@ -9,8 +9,8 @@ commit = "fa44e5194060305576514d59b850353643afbfc8" # v14.0.0 license = "MIT OR Apache-2.0" [build] -baseline = "cargo build --release --manifest-path workspace/Cargo.toml" -binary = "workspace/target/release/tokei" +baseline = "cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml" +binary = "targets/tokei/workspace/target/release/tokei" [bench] # Single rayon thread: measurement first (SPEC conventions); the @@ -19,5 +19,10 @@ command = "RAYON_NUM_THREADS=1 {binary} --sort code corpora/tokei/input > /dev/n workload = "8 replicas of pinned progit + comrak/src trees, 221 MB, 4360 files, single rayon thread" [gates] -tests = "cargo test --release --manifest-path workspace/Cargo.toml" +tests = "cargo test --release --manifest-path targets/tokei/workspace/Cargo.toml" golden = "RAYON_NUM_THREADS=1 {binary} --sort code corpora/tokei/input" + +[corpus] +manifest = "corpora/tokei/MANIFEST.sha256" +root = "corpora/tokei/input/copy1" +golden_sha256 = "corpora/tokei/GOLDEN.sha256"