Merge pull request #5 from bdbrown4/claude/test-wsk42i

.gitignore

@@ -234,3 +234,4 @@ targets/comrak/asan*/
 targets/tokei/candidate-*/
 targets/tokei/asan/
 !results/calibration/*.json
+targets/*/baseline/

CLAUDE.md

@@ -56,6 +56,7 @@ just compare <a> <b>        # interleaved A/B of two shell commands, bootstrap C
 just gates <target>         # corpus pin + upstream tests + golden replay
 just pin-check <target>     # verify corpus MANIFEST.sha256
 just calibrate <cmd> <out>  # automated A/A + regression-injection calibration
+just verdict <t> <bin> ...  # gates + bench vs pristine-rebuilt baseline + ledger row
 ```
 
 ## Definition of done — one optimization attempt

Cargo.lock

@@ -558,6 +558,18 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "verdict"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "bench-runner",
+ "clap",
+ "diff-test",
+ "ledger",
+ "serde_json",
+]
+
 [[package]]
 name = "version_check"
 version = "0.9.5"

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "crates/diff-test",
     "crates/ledger",
     "crates/report",
+    "crates/verdict",
 ]
 
 [workspace.package]

config/accept.toml

@@ -18,3 +18,8 @@ confidence = 0.95
 
 # Fixed PRNG seed so calibration runs are reproducible.
 bootstrap_seed = 1887343589
+
+# Machine-specific CPU pinning applied identically to both bench sides.
+# Calibration evidence for this machine shows pinning tightens the A/A
+# CI ~5x (results/calibration/). Empty string disables.
+pin_prefix = "taskset -c 2"

crates/bench-runner/src/config.rs

@@ -18,6 +18,11 @@ pub struct AcceptConfig {
     pub confidence: f64,
     /// PRNG seed for the bootstrap (reproducible calibration).
     pub bootstrap_seed: u64,
+    /// Machine-specific command prefix for CPU pinning (e.g.
+    /// "taskset -c 2"); empty disables pinning. Applied by verdict to
+    /// both sides identically.
+    #[serde(default)]
+    pub pin_prefix: String,
 }
 
 impl Default for AcceptConfig {
@@ -29,6 +34,7 @@ impl Default for AcceptConfig {
             bootstrap_iters: 10_000,
             confidence: 0.95,
             bootstrap_seed: 0x707E_17E5,
+            pin_prefix: String::new(),
         }
     }
 }

crates/bench-runner/src/stats.rs

@@ -80,10 +80,37 @@ pub fn bootstrap_ratio_ci(
     }
 }
 
+/// Percentile-method bootstrap CI of a single sample's median, for
+/// per-side evidence in ledger rows.
+pub fn bootstrap_median_ci(sample: &[f64], iters: usize, confidence: f64, seed: u64) -> (f64, f64, f64) {
+    assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence));
+    let mut rng = XorShift64::new(seed);
+    let mut medians = Vec::with_capacity(iters);
+    let mut resample = vec![0.0; sample.len()];
+    for _ in 0..iters {
+        for slot in resample.iter_mut() {
+            *slot = sample[rng.index(sample.len())];
+        }
+        medians.push(median(&resample));
+    }
+    medians.sort_by(|a, b| a.partial_cmp(b).expect("NaN median"));
+    let alpha = (1.0 - confidence) / 2.0;
+    let lo = medians[((iters as f64) * alpha).floor() as usize];
+    let hi = medians[(((iters as f64) * (1.0 - alpha)).ceil() as usize).min(iters - 1)];
+    (median(sample), lo, hi)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn median_ci_brackets_median() {
+        let s: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
+        let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9);
+        assert!(lo <= m && m <= hi);
+    }
+
     #[test]
     fn median_odd_and_even() {
         assert_eq!(median(&[3.0, 1.0, 2.0]), 2.0);

crates/diff-test/src/lib.rs

@@ -7,6 +7,7 @@
 //! per-target equivalence policy parser (byte-identical unless a target
 //! declares an explicit FP tolerance).
 
+pub mod orchestrate;
 pub mod pin;
 pub mod target;
 pub mod policy;

crates/diff-test/src/main.rs

@@ -1,15 +1,13 @@
-//! Gate orchestration: corpus pin → upstream tests → golden replay.
+//! Gate orchestration CLI: corpus pin → upstream tests → golden replay.
-//! Refuses to run anything if the corpus manifest mismatches (SPEC §3.2).
 //! Differential fuzz and sanitizers remain per-attempt manual steps in
 //! Phase 1 and are reported as skipped with a reason.
 
 use anyhow::Result;
 use clap::Parser;
-use diff_test::target::{expected_golden_hash, TargetSpec};
+use diff_test::orchestrate::run_core_gates;
-use diff_test::{all_passed, pin, GateLayer, GateOutcome};
+use diff_test::target::TargetSpec;
-use sha2::{Digest, Sha256};
+use diff_test::{all_passed, GateOutcome};
 use std::path::PathBuf;
-use std::process::{Command, Stdio};
 
 #[derive(Parser)]
 #[command(name = "diff-test", about = "hotpath equivalence-gate orchestrator")]
@@ -21,67 +19,16 @@ struct Cli {
     root: PathBuf,
 }
 
-fn sh(root: &PathBuf, cmd: &str) -> Result<std::process::Output> {
-    Ok(Command::new("sh")
-        .arg("-c")
-        .arg(cmd)
-        .current_dir(root)
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .output()?)
-}
-
 fn main() -> Result<()> {
     let cli = Cli::parse();
     let root = cli.root.canonicalize()?;
     let spec = TargetSpec::load(&root, &cli.target)?;
-    let mut results: Vec<(GateLayer, GateOutcome)> = Vec::new();
+    let results = run_core_gates(&root, &spec, &spec.build.binary)?;
-
-    // Corpus pin is a precondition, not a gate: mismatch = refuse to run.
-    let verified = pin::verify_manifest(&root.join(&spec.corpus.manifest), &root.join(&spec.corpus.root))?;
-    println!("corpus pin: {} files verified against {}", verified, spec.corpus.manifest.display());
-
-    print!("upstream tests ... ");
-    let t = sh(&root, &spec.gates.tests)?;
-    let tests_outcome = if t.status.success() {
-        println!("PASS");
-        GateOutcome::Passed
-    } else {
-        println!("FAIL");
-        GateOutcome::Failed {
-            detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(),
-        }
-    };
-    results.push((GateLayer::UpstreamTests, tests_outcome));
-
-    print!("golden replay ... ");
-    let golden_cmd = spec.gates.golden.replace("{binary}", &spec.build.binary);
-    let g = sh(&root, &golden_cmd)?;
-    let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
-    let actual = format!("{:x}", Sha256::digest(&g.stdout));
-    let golden_outcome = if g.status.success() && actual == expected {
-        println!("PASS (stdout sha256 = pinned golden)");
-        GateOutcome::Passed
-    } else {
-        println!("FAIL");
-        GateOutcome::Failed {
-            detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()),
-        }
-    };
-    results.push((GateLayer::GoldenReplay, golden_outcome));
-
-    results.push((
-        GateLayer::DifferentialFuzz,
-        GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into() },
-    ));
-    results.push((
-        GateLayer::Sanitizers,
-        GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into() },
-    ));
-
     for (layer, outcome) in &results {
-        if let GateOutcome::Skipped { reason } = outcome {
+        match outcome {
-            println!("{layer:?}: skipped — {reason}");
+            GateOutcome::Passed => println!("{layer:?}: PASS"),
+            GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"),
+            GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
         }
     }
     if all_passed(&results) {

crates/diff-test/src/orchestrate.rs

@@ -0,0 +1,78 @@
+//! Core gate sequence, callable from the diff-test CLI and from verdict:
+//! corpus pin (refuses on mismatch) → upstream tests → golden replay.
+
+use crate::target::{expected_golden_hash, TargetSpec};
+use crate::{pin, GateLayer, GateOutcome};
+use sha2::{Digest, Sha256};
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+fn sh(root: &Path, cmd: &str) -> anyhow::Result<std::process::Output> {
+    Ok(Command::new("sh")
+        .arg("-c")
+        .arg(cmd)
+        .current_dir(root)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .output()?)
+}
+
+/// Run the mechanical gates for `target`, testing `binary` (repo-root-
+/// relative) in the golden replay. Corpus-pin mismatch is an error, not
+/// a gate failure: nothing runs on a tampered corpus.
+pub fn run_core_gates(
+    root: &Path,
+    spec: &TargetSpec,
+    binary: &str,
+) -> anyhow::Result<Vec<(GateLayer, GateOutcome)>> {
+    let verified = pin::verify_manifest(
+        &root.join(&spec.corpus.manifest),
+        &root.join(&spec.corpus.root),
+    )?;
+    eprintln!(
+        "corpus pin: {verified} files verified against {}",
+        spec.corpus.manifest.display()
+    );
+
+    let mut results = Vec::new();
+
+    let t = sh(root, &spec.gates.tests)?;
+    results.push((
+        GateLayer::UpstreamTests,
+        if t.status.success() {
+            GateOutcome::Passed
+        } else {
+            GateOutcome::Failed {
+                detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(),
+            }
+        },
+    ));
+
+    let g = sh(root, &spec.gates.golden.replace("{binary}", binary))?;
+    let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
+    let actual = format!("{:x}", Sha256::digest(&g.stdout));
+    results.push((
+        GateLayer::GoldenReplay,
+        if g.status.success() && actual == expected {
+            GateOutcome::Passed
+        } else {
+            GateOutcome::Failed {
+                detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()),
+            }
+        },
+    ));
+
+    results.push((
+        GateLayer::DifferentialFuzz,
+        GateOutcome::Skipped {
+            reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into(),
+        },
+    ));
+    results.push((
+        GateLayer::Sanitizers,
+        GateOutcome::Skipped {
+            reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into(),
+        },
+    ));
+    Ok(results)
+}

crates/diff-test/src/target.rs

@@ -9,10 +9,19 @@ use std::path::{Path, PathBuf};
 pub struct TargetSpec {
     pub source: Source,
     pub build: Build,
+    pub bench: Bench,
     pub gates: Gates,
     pub corpus: Corpus,
 }
 
+#[derive(Debug, Deserialize)]
+pub struct Bench {
+    /// Bench command; `{binary}` substituted per side. Run from repo root.
+    pub command: String,
+    /// Human-readable workload statement, printed with every number.
+    pub workload: String,
+}
+
 #[derive(Debug, Deserialize)]
 pub struct Source {
     pub repo: String,

crates/verdict/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "verdict"
+description = "One-shot attempt verdict: equivalence gates + interleaved A/B bench + append-only ledger row"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+bench-runner = { path = "../bench-runner" }
+clap.workspace = true
+diff-test = { path = "../diff-test" }
+ledger = { path = "../ledger" }
+serde_json.workspace = true

crates/verdict/src/main.rs

@@ -0,0 +1,220 @@
+//! `just verdict`: the whole attempt pipeline in one command.
+//! Gates on the candidate → interleaved A/B bench vs the baseline →
+//! verdict by the CI-lower-bound rule → append-only ledger row.
+//! Any gate failure is rejected-gate and the bench is skipped; the
+//! --needs-human-review flag caps the verdict for FP/concurrency/
+//! sanitizer-flagged attempts (SPEC §8) — it never upgrades one.
+
+use anyhow::Result;
+use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
+use clap::Parser;
+use diff_test::{all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome};
+use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict};
+use std::path::PathBuf;
+
+#[derive(Parser)]
+#[command(name = "verdict", about = "gates + bench + ledger row for one optimization attempt")]
+struct Cli {
+    /// Target name under targets/.
+    target: String,
+    /// Pristine baseline binary (repo-root-relative). Prefer
+    /// --rebuild-baseline: workspace paths get silently rebuilt from the
+    /// patched tree by test runs (observed: phase1-verdict-smoke-001).
+    #[arg(long, conflicts_with = "rebuild_baseline")]
+    baseline_bin: Option<String>,
+    /// Rebuild the baseline from the pristine checkout (patch stashed,
+    /// built to targets/<name>/baseline/, patch restored). SPEC §3.1.
+    #[arg(long)]
+    rebuild_baseline: bool,
+    /// Candidate binary (repo-root-relative); also used in golden replay.
+    #[arg(long)]
+    candidate_bin: String,
+    #[arg(long)]
+    run_id: String,
+    #[arg(long)]
+    playbook_class: u8,
+    #[arg(long)]
+    hypothesis: String,
+    #[arg(long)]
+    hotspot: String,
+    /// Unified-diff file recorded in the ledger (optional for build-config attempts).
+    #[arg(long)]
+    patch_file: Option<PathBuf>,
+    /// Force the needs-human-review verdict tier (FP/concurrency/UB-adjacent).
+    #[arg(long)]
+    needs_human_review: bool,
+    #[arg(long, default_value = "results/ledger.sqlite")]
+    db: PathBuf,
+    #[arg(long, default_value = "config/accept.toml")]
+    config: PathBuf,
+    #[arg(long, default_value = ".")]
+    root: PathBuf,
+}
+
+/// Stash any workspace patch, build the baseline into the isolated
+/// targets/<name>/baseline/ dir, restore the patch. Serialized within
+/// this process — nothing else may touch the tree while this runs.
+fn rebuild_pristine_baseline(
+    root: &std::path::Path,
+    target: &str,
+    spec: &TargetSpec,
+) -> Result<String> {
+    let ws = format!("targets/{target}/workspace");
+    let dirty = !std::process::Command::new("git")
+        .args(["-C", &ws, "status", "--porcelain"])
+        .output()?
+        .stdout
+        .is_empty();
+    if dirty {
+        anyhow::ensure!(
+            std::process::Command::new("git")
+                .args(["-C", &ws, "stash", "--include-untracked", "-q"])
+                .status()?
+                .success(),
+            "git stash failed"
+        );
+    }
+    let build = std::process::Command::new("sh")
+        .arg("-c")
+        .arg(&spec.build.baseline)
+        .current_dir(root)
+        .env("CARGO_TARGET_DIR", root.join(format!("targets/{target}/baseline")))
+        .status();
+    if dirty {
+        anyhow::ensure!(
+            std::process::Command::new("git")
+                .args(["-C", &ws, "stash", "pop", "-q"])
+                .status()?
+                .success(),
+            "git stash pop failed — workspace patch may be stranded in stash"
+        );
+    }
+    anyhow::ensure!(build?.success(), "baseline build failed");
+    let bin_name = std::path::Path::new(&spec.build.binary)
+        .file_name()
+        .and_then(|n| n.to_str())
+        .ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
+    let baseline = format!("targets/{target}/baseline/release/{bin_name}");
+    anyhow::ensure!(root.join(&baseline).exists(), "baseline binary missing: {baseline}");
+    println!("baseline rebuilt from pristine checkout: {baseline}");
+    Ok(baseline)
+}
+
+fn now_utc() -> String {
+    let out = std::process::Command::new("date")
+        .args(["-u", "+%Y-%m-%dT%H:%M:%SZ"])
+        .output();
+    match out {
+        Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).trim().to_string(),
+        _ => "unknown".to_string(),
+    }
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let root = cli.root.canonicalize()?;
+    std::env::set_current_dir(&root)?;
+    let spec = TargetSpec::load(&root, &cli.target)?;
+    let cfg = AcceptConfig::load(&root.join(&cli.config))?;
+    let started = std::time::Instant::now();
+
+    let baseline_bin = if cli.rebuild_baseline {
+        rebuild_pristine_baseline(&root, &cli.target, &spec)?
+    } else {
+        cli.baseline_bin.clone().ok_or_else(|| {
+            anyhow::anyhow!("pass --rebuild-baseline (preferred) or --baseline-bin")
+        })?
+    };
+
+    // Gates run on the candidate binary.
+    let gates = run_core_gates(&root, &spec, &cli.candidate_bin)?;
+    for (layer, outcome) in &gates {
+        match outcome {
+            GateOutcome::Passed => println!("{layer:?}: PASS"),
+            GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"),
+            GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
+        }
+    }
+    let gate_results = GateResults {
+        upstream_tests: matches!(gates[0].1, GateOutcome::Passed),
+        golden_replay: matches!(gates[1].1, GateOutcome::Passed),
+        fuzz_iters: 0,
+        fuzz_divergence: false,
+        sanitizers_clean: false,
+    };
+    debug_assert!(matches!(gates[0].0, GateLayer::UpstreamTests));
+
+    let pin = &cfg.pin_prefix;
+    let wrap = |bin: &str| {
+        let cmd = spec.bench.command.replace("{binary}", bin);
+        if pin.is_empty() { format!("sh -c \"{cmd}\"") } else { format!("{pin} sh -c \"{cmd}\"") }
+    };
+
+    let (verdict, bench) = if !all_passed(&gates) {
+        println!("verdict: rejected-gate (bench skipped)");
+        (Verdict::RejectedGate, None)
+    } else {
+        println!(
+            "bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'",
+            cfg.runs_per_side, cfg.warmup_runs, pin
+        );
+        let samples = exec::run_interleaved(
+            &wrap(&baseline_bin), &wrap(&cli.candidate_bin),
+            cfg.runs_per_side, cfg.warmup_runs, 0.0,
+        )?;
+        let ratio = stats::bootstrap_ratio_ci(
+            &samples.baseline_s, &samples.candidate_s,
+            cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
+        );
+        let (bm, blo, bhi) = stats::bootstrap_median_ci(
+            &samples.baseline_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
+        );
+        let (cm, clo, chi) = stats::bootstrap_median_ci(
+            &samples.candidate_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
+        );
+        println!(
+            "speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
+            ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload
+        );
+        let mut v = decide(ratio, cfg.threshold);
+        if cli.needs_human_review && v == Verdict::Accepted {
+            v = Verdict::NeedsHumanReview;
+        }
+        let fp = EnvFingerprint::collect(pin, "system-default");
+        (v, Some(BenchEvidence {
+            baseline_median: bm, baseline_ci: (blo, bhi),
+            candidate_median: cm, candidate_ci: (clo, chi),
+            speedup_median: ratio.median, speedup_ci: (ratio.lo, ratio.hi),
+            env_fingerprint: serde_json::json!({
+                "fingerprint": fp,
+                "workload": spec.bench.workload,
+                "target_commit": spec.source.commit,
+                "gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
+            }),
+        }))
+    };
+
+    let patch = match &cli.patch_file {
+        Some(p) => std::fs::read_to_string(p)?,
+        None => "(no source patch)".into(),
+    };
+    let attempt = Attempt {
+        run_id: cli.run_id.clone(),
+        timestamp: now_utc(),
+        target: cli.target.clone(),
+        target_commit: spec.source.commit.clone(),
+        phase: 1,
+        hotspot: cli.hotspot,
+        playbook_class: cli.playbook_class,
+        hypothesis: cli.hypothesis,
+        patch,
+        gates: gate_results,
+        bench,
+        verdict,
+        tokens_spent: 0,
+        wall_time_s: started.elapsed().as_secs_f64(),
+    };
+    Ledger::open(&root.join(&cli.db))?.record(&attempt)?;
+    println!("verdict: {} (ledger row {})", attempt.verdict.as_str(), attempt.run_id);
+    Ok(())
+}

justfile

@@ -38,3 +38,12 @@ calibrate cmd out sessions="20":
 # upstream tests -> golden replay. Fuzz/sanitizers stay per-attempt.
 gates target:
     cargo run -p diff-test -- {{target}}
+
+# One-shot attempt verdict: gates on candidate -> interleaved A/B vs a
+# pristine-rebuilt baseline -> ledger row. Extra flags pass through
+# (--patch-file, --needs-human-review, --baseline-bin).
+verdict target candidate run_id class hypothesis hotspot *flags:
+    cargo run -p verdict -- {{target}} --rebuild-baseline \
+        --candidate-bin "{{candidate}}" --run-id "{{run_id}}" \
+        --playbook-class {{class}} --hypothesis "{{hypothesis}}" \
+        --hotspot "{{hotspot}}" {{flags}}