diff --git a/.gitignore b/.gitignore index 6cc2103..87a9738 100644 --- a/.gitignore +++ b/.gitignore @@ -234,3 +234,4 @@ targets/comrak/asan*/ targets/tokei/candidate-*/ targets/tokei/asan/ !results/calibration/*.json +targets/*/baseline/ diff --git a/CLAUDE.md b/CLAUDE.md index 253cb9c..216fab2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,6 +56,7 @@ just compare # interleaved A/B of two shell commands, bootstrap C just gates # corpus pin + upstream tests + golden replay just pin-check # verify corpus MANIFEST.sha256 just calibrate # automated A/A + regression-injection calibration +just verdict ... # gates + bench vs pristine-rebuilt baseline + ledger row ``` ## Definition of done — one optimization attempt diff --git a/Cargo.lock b/Cargo.lock index 8710e86..4daf4fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,6 +558,18 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "verdict" +version = "0.1.0" +dependencies = [ + "anyhow", + "bench-runner", + "clap", + "diff-test", + "ledger", + "serde_json", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index a0f93b8..f52d276 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/diff-test", "crates/ledger", "crates/report", + "crates/verdict", ] [workspace.package] diff --git a/config/accept.toml b/config/accept.toml index 5da4e72..8497e82 100644 --- a/config/accept.toml +++ b/config/accept.toml @@ -18,3 +18,8 @@ confidence = 0.95 # Fixed PRNG seed so calibration runs are reproducible. bootstrap_seed = 1887343589 + +# Machine-specific CPU pinning applied identically to both bench sides. +# Calibration evidence for this machine shows pinning tightens the A/A +# CI ~5x (results/calibration/). Empty string disables. +pin_prefix = "taskset -c 2" diff --git a/crates/bench-runner/src/config.rs b/crates/bench-runner/src/config.rs index 437ff91..6f6d822 100644 --- a/crates/bench-runner/src/config.rs +++ b/crates/bench-runner/src/config.rs @@ -18,6 +18,11 @@ pub struct AcceptConfig { pub confidence: f64, /// PRNG seed for the bootstrap (reproducible calibration). pub bootstrap_seed: u64, + /// Machine-specific command prefix for CPU pinning (e.g. + /// "taskset -c 2"); empty disables pinning. Applied by verdict to + /// both sides identically. + #[serde(default)] + pub pin_prefix: String, } impl Default for AcceptConfig { @@ -29,6 +34,7 @@ impl Default for AcceptConfig { bootstrap_iters: 10_000, confidence: 0.95, bootstrap_seed: 0x707E_17E5, + pin_prefix: String::new(), } } } diff --git a/crates/bench-runner/src/stats.rs b/crates/bench-runner/src/stats.rs index e95180b..53576e0 100644 --- a/crates/bench-runner/src/stats.rs +++ b/crates/bench-runner/src/stats.rs @@ -80,10 +80,37 @@ pub fn bootstrap_ratio_ci( } } +/// Percentile-method bootstrap CI of a single sample's median, for +/// per-side evidence in ledger rows. +pub fn bootstrap_median_ci(sample: &[f64], iters: usize, confidence: f64, seed: u64) -> (f64, f64, f64) { + assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence)); + let mut rng = XorShift64::new(seed); + let mut medians = Vec::with_capacity(iters); + let mut resample = vec![0.0; sample.len()]; + for _ in 0..iters { + for slot in resample.iter_mut() { + *slot = sample[rng.index(sample.len())]; + } + medians.push(median(&resample)); + } + medians.sort_by(|a, b| a.partial_cmp(b).expect("NaN median")); + let alpha = (1.0 - confidence) / 2.0; + let lo = medians[((iters as f64) * alpha).floor() as usize]; + let hi = medians[(((iters as f64) * (1.0 - alpha)).ceil() as usize).min(iters - 1)]; + (median(sample), lo, hi) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn median_ci_brackets_median() { + let s: Vec = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect(); + let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9); + assert!(lo <= m && m <= hi); + } + #[test] fn median_odd_and_even() { assert_eq!(median(&[3.0, 1.0, 2.0]), 2.0); diff --git a/crates/diff-test/src/lib.rs b/crates/diff-test/src/lib.rs index 0c44a8c..131e8db 100644 --- a/crates/diff-test/src/lib.rs +++ b/crates/diff-test/src/lib.rs @@ -7,6 +7,7 @@ //! per-target equivalence policy parser (byte-identical unless a target //! declares an explicit FP tolerance). +pub mod orchestrate; pub mod pin; pub mod target; pub mod policy; diff --git a/crates/diff-test/src/main.rs b/crates/diff-test/src/main.rs index 678356b..8dc5f77 100644 --- a/crates/diff-test/src/main.rs +++ b/crates/diff-test/src/main.rs @@ -1,15 +1,13 @@ -//! Gate orchestration: corpus pin → upstream tests → golden replay. -//! Refuses to run anything if the corpus manifest mismatches (SPEC §3.2). +//! Gate orchestration CLI: corpus pin → upstream tests → golden replay. //! Differential fuzz and sanitizers remain per-attempt manual steps in //! Phase 1 and are reported as skipped with a reason. use anyhow::Result; use clap::Parser; -use diff_test::target::{expected_golden_hash, TargetSpec}; -use diff_test::{all_passed, pin, GateLayer, GateOutcome}; -use sha2::{Digest, Sha256}; +use diff_test::orchestrate::run_core_gates; +use diff_test::target::TargetSpec; +use diff_test::{all_passed, GateOutcome}; use std::path::PathBuf; -use std::process::{Command, Stdio}; #[derive(Parser)] #[command(name = "diff-test", about = "hotpath equivalence-gate orchestrator")] @@ -21,67 +19,16 @@ struct Cli { root: PathBuf, } -fn sh(root: &PathBuf, cmd: &str) -> Result { - Ok(Command::new("sh") - .arg("-c") - .arg(cmd) - .current_dir(root) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output()?) -} - fn main() -> Result<()> { let cli = Cli::parse(); let root = cli.root.canonicalize()?; let spec = TargetSpec::load(&root, &cli.target)?; - let mut results: Vec<(GateLayer, GateOutcome)> = Vec::new(); - - // Corpus pin is a precondition, not a gate: mismatch = refuse to run. - let verified = pin::verify_manifest(&root.join(&spec.corpus.manifest), &root.join(&spec.corpus.root))?; - println!("corpus pin: {} files verified against {}", verified, spec.corpus.manifest.display()); - - print!("upstream tests ... "); - let t = sh(&root, &spec.gates.tests)?; - let tests_outcome = if t.status.success() { - println!("PASS"); - GateOutcome::Passed - } else { - println!("FAIL"); - GateOutcome::Failed { - detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(), - } - }; - results.push((GateLayer::UpstreamTests, tests_outcome)); - - print!("golden replay ... "); - let golden_cmd = spec.gates.golden.replace("{binary}", &spec.build.binary); - let g = sh(&root, &golden_cmd)?; - let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?; - let actual = format!("{:x}", Sha256::digest(&g.stdout)); - let golden_outcome = if g.status.success() && actual == expected { - println!("PASS (stdout sha256 = pinned golden)"); - GateOutcome::Passed - } else { - println!("FAIL"); - GateOutcome::Failed { - detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()), - } - }; - results.push((GateLayer::GoldenReplay, golden_outcome)); - - results.push(( - GateLayer::DifferentialFuzz, - GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into() }, - )); - results.push(( - GateLayer::Sanitizers, - GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into() }, - )); - + let results = run_core_gates(&root, &spec, &spec.build.binary)?; for (layer, outcome) in &results { - if let GateOutcome::Skipped { reason } = outcome { - println!("{layer:?}: skipped — {reason}"); + match outcome { + GateOutcome::Passed => println!("{layer:?}: PASS"), + GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"), + GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"), } } if all_passed(&results) { diff --git a/crates/diff-test/src/orchestrate.rs b/crates/diff-test/src/orchestrate.rs new file mode 100644 index 0000000..b140b77 --- /dev/null +++ b/crates/diff-test/src/orchestrate.rs @@ -0,0 +1,78 @@ +//! Core gate sequence, callable from the diff-test CLI and from verdict: +//! corpus pin (refuses on mismatch) → upstream tests → golden replay. + +use crate::target::{expected_golden_hash, TargetSpec}; +use crate::{pin, GateLayer, GateOutcome}; +use sha2::{Digest, Sha256}; +use std::path::Path; +use std::process::{Command, Stdio}; + +fn sh(root: &Path, cmd: &str) -> anyhow::Result { + Ok(Command::new("sh") + .arg("-c") + .arg(cmd) + .current_dir(root) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output()?) +} + +/// Run the mechanical gates for `target`, testing `binary` (repo-root- +/// relative) in the golden replay. Corpus-pin mismatch is an error, not +/// a gate failure: nothing runs on a tampered corpus. +pub fn run_core_gates( + root: &Path, + spec: &TargetSpec, + binary: &str, +) -> anyhow::Result> { + let verified = pin::verify_manifest( + &root.join(&spec.corpus.manifest), + &root.join(&spec.corpus.root), + )?; + eprintln!( + "corpus pin: {verified} files verified against {}", + spec.corpus.manifest.display() + ); + + let mut results = Vec::new(); + + let t = sh(root, &spec.gates.tests)?; + results.push(( + GateLayer::UpstreamTests, + if t.status.success() { + GateOutcome::Passed + } else { + GateOutcome::Failed { + detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(), + } + }, + )); + + let g = sh(root, &spec.gates.golden.replace("{binary}", binary))?; + let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?; + let actual = format!("{:x}", Sha256::digest(&g.stdout)); + results.push(( + GateLayer::GoldenReplay, + if g.status.success() && actual == expected { + GateOutcome::Passed + } else { + GateOutcome::Failed { + detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()), + } + }, + )); + + results.push(( + GateLayer::DifferentialFuzz, + GateOutcome::Skipped { + reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into(), + }, + )); + results.push(( + GateLayer::Sanitizers, + GateOutcome::Skipped { + reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into(), + }, + )); + Ok(results) +} diff --git a/crates/diff-test/src/target.rs b/crates/diff-test/src/target.rs index e11b9bc..0b7a4f4 100644 --- a/crates/diff-test/src/target.rs +++ b/crates/diff-test/src/target.rs @@ -9,10 +9,19 @@ use std::path::{Path, PathBuf}; pub struct TargetSpec { pub source: Source, pub build: Build, + pub bench: Bench, pub gates: Gates, pub corpus: Corpus, } +#[derive(Debug, Deserialize)] +pub struct Bench { + /// Bench command; `{binary}` substituted per side. Run from repo root. + pub command: String, + /// Human-readable workload statement, printed with every number. + pub workload: String, +} + #[derive(Debug, Deserialize)] pub struct Source { pub repo: String, diff --git a/crates/verdict/Cargo.toml b/crates/verdict/Cargo.toml new file mode 100644 index 0000000..e3373b7 --- /dev/null +++ b/crates/verdict/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "verdict" +description = "One-shot attempt verdict: equivalence gates + interleaved A/B bench + append-only ledger row" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +bench-runner = { path = "../bench-runner" } +clap.workspace = true +diff-test = { path = "../diff-test" } +ledger = { path = "../ledger" } +serde_json.workspace = true diff --git a/crates/verdict/src/main.rs b/crates/verdict/src/main.rs new file mode 100644 index 0000000..999ad3c --- /dev/null +++ b/crates/verdict/src/main.rs @@ -0,0 +1,220 @@ +//! `just verdict`: the whole attempt pipeline in one command. +//! Gates on the candidate → interleaved A/B bench vs the baseline → +//! verdict by the CI-lower-bound rule → append-only ledger row. +//! Any gate failure is rejected-gate and the bench is skipped; the +//! --needs-human-review flag caps the verdict for FP/concurrency/ +//! sanitizer-flagged attempts (SPEC §8) — it never upgrades one. + +use anyhow::Result; +use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats}; +use clap::Parser; +use diff_test::{all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome}; +use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(name = "verdict", about = "gates + bench + ledger row for one optimization attempt")] +struct Cli { + /// Target name under targets/. + target: String, + /// Pristine baseline binary (repo-root-relative). Prefer + /// --rebuild-baseline: workspace paths get silently rebuilt from the + /// patched tree by test runs (observed: phase1-verdict-smoke-001). + #[arg(long, conflicts_with = "rebuild_baseline")] + baseline_bin: Option, + /// Rebuild the baseline from the pristine checkout (patch stashed, + /// built to targets//baseline/, patch restored). SPEC §3.1. + #[arg(long)] + rebuild_baseline: bool, + /// Candidate binary (repo-root-relative); also used in golden replay. + #[arg(long)] + candidate_bin: String, + #[arg(long)] + run_id: String, + #[arg(long)] + playbook_class: u8, + #[arg(long)] + hypothesis: String, + #[arg(long)] + hotspot: String, + /// Unified-diff file recorded in the ledger (optional for build-config attempts). + #[arg(long)] + patch_file: Option, + /// Force the needs-human-review verdict tier (FP/concurrency/UB-adjacent). + #[arg(long)] + needs_human_review: bool, + #[arg(long, default_value = "results/ledger.sqlite")] + db: PathBuf, + #[arg(long, default_value = "config/accept.toml")] + config: PathBuf, + #[arg(long, default_value = ".")] + root: PathBuf, +} + +/// Stash any workspace patch, build the baseline into the isolated +/// targets//baseline/ dir, restore the patch. Serialized within +/// this process — nothing else may touch the tree while this runs. +fn rebuild_pristine_baseline( + root: &std::path::Path, + target: &str, + spec: &TargetSpec, +) -> Result { + let ws = format!("targets/{target}/workspace"); + let dirty = !std::process::Command::new("git") + .args(["-C", &ws, "status", "--porcelain"]) + .output()? + .stdout + .is_empty(); + if dirty { + anyhow::ensure!( + std::process::Command::new("git") + .args(["-C", &ws, "stash", "--include-untracked", "-q"]) + .status()? + .success(), + "git stash failed" + ); + } + let build = std::process::Command::new("sh") + .arg("-c") + .arg(&spec.build.baseline) + .current_dir(root) + .env("CARGO_TARGET_DIR", root.join(format!("targets/{target}/baseline"))) + .status(); + if dirty { + anyhow::ensure!( + std::process::Command::new("git") + .args(["-C", &ws, "stash", "pop", "-q"]) + .status()? + .success(), + "git stash pop failed — workspace patch may be stranded in stash" + ); + } + anyhow::ensure!(build?.success(), "baseline build failed"); + let bin_name = std::path::Path::new(&spec.build.binary) + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?; + let baseline = format!("targets/{target}/baseline/release/{bin_name}"); + anyhow::ensure!(root.join(&baseline).exists(), "baseline binary missing: {baseline}"); + println!("baseline rebuilt from pristine checkout: {baseline}"); + Ok(baseline) +} + +fn now_utc() -> String { + let out = std::process::Command::new("date") + .args(["-u", "+%Y-%m-%dT%H:%M:%SZ"]) + .output(); + match out { + Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).trim().to_string(), + _ => "unknown".to_string(), + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let root = cli.root.canonicalize()?; + std::env::set_current_dir(&root)?; + let spec = TargetSpec::load(&root, &cli.target)?; + let cfg = AcceptConfig::load(&root.join(&cli.config))?; + let started = std::time::Instant::now(); + + let baseline_bin = if cli.rebuild_baseline { + rebuild_pristine_baseline(&root, &cli.target, &spec)? + } else { + cli.baseline_bin.clone().ok_or_else(|| { + anyhow::anyhow!("pass --rebuild-baseline (preferred) or --baseline-bin") + })? + }; + + // Gates run on the candidate binary. + let gates = run_core_gates(&root, &spec, &cli.candidate_bin)?; + for (layer, outcome) in &gates { + match outcome { + GateOutcome::Passed => println!("{layer:?}: PASS"), + GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"), + GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"), + } + } + let gate_results = GateResults { + upstream_tests: matches!(gates[0].1, GateOutcome::Passed), + golden_replay: matches!(gates[1].1, GateOutcome::Passed), + fuzz_iters: 0, + fuzz_divergence: false, + sanitizers_clean: false, + }; + debug_assert!(matches!(gates[0].0, GateLayer::UpstreamTests)); + + let pin = &cfg.pin_prefix; + let wrap = |bin: &str| { + let cmd = spec.bench.command.replace("{binary}", bin); + if pin.is_empty() { format!("sh -c \"{cmd}\"") } else { format!("{pin} sh -c \"{cmd}\"") } + }; + + let (verdict, bench) = if !all_passed(&gates) { + println!("verdict: rejected-gate (bench skipped)"); + (Verdict::RejectedGate, None) + } else { + println!( + "bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'", + cfg.runs_per_side, cfg.warmup_runs, pin + ); + let samples = exec::run_interleaved( + &wrap(&baseline_bin), &wrap(&cli.candidate_bin), + cfg.runs_per_side, cfg.warmup_runs, 0.0, + )?; + let ratio = stats::bootstrap_ratio_ci( + &samples.baseline_s, &samples.candidate_s, + cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed, + ); + let (bm, blo, bhi) = stats::bootstrap_median_ci( + &samples.baseline_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed, + ); + let (cm, clo, chi) = stats::bootstrap_median_ci( + &samples.candidate_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed, + ); + println!( + "speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}", + ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload + ); + let mut v = decide(ratio, cfg.threshold); + if cli.needs_human_review && v == Verdict::Accepted { + v = Verdict::NeedsHumanReview; + } + let fp = EnvFingerprint::collect(pin, "system-default"); + (v, Some(BenchEvidence { + baseline_median: bm, baseline_ci: (blo, bhi), + candidate_median: cm, candidate_ci: (clo, chi), + speedup_median: ratio.median, speedup_ci: (ratio.lo, ratio.hi), + env_fingerprint: serde_json::json!({ + "fingerprint": fp, + "workload": spec.bench.workload, + "target_commit": spec.source.commit, + "gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1", + }), + })) + }; + + let patch = match &cli.patch_file { + Some(p) => std::fs::read_to_string(p)?, + None => "(no source patch)".into(), + }; + let attempt = Attempt { + run_id: cli.run_id.clone(), + timestamp: now_utc(), + target: cli.target.clone(), + target_commit: spec.source.commit.clone(), + phase: 1, + hotspot: cli.hotspot, + playbook_class: cli.playbook_class, + hypothesis: cli.hypothesis, + patch, + gates: gate_results, + bench, + verdict, + tokens_spent: 0, + wall_time_s: started.elapsed().as_secs_f64(), + }; + Ledger::open(&root.join(&cli.db))?.record(&attempt)?; + println!("verdict: {} (ledger row {})", attempt.verdict.as_str(), attempt.run_id); + Ok(()) +} diff --git a/justfile b/justfile index f0fcc26..b394e2e 100644 --- a/justfile +++ b/justfile @@ -38,3 +38,12 @@ calibrate cmd out sessions="20": # upstream tests -> golden replay. Fuzz/sanitizers stay per-attempt. gates target: cargo run -p diff-test -- {{target}} + +# One-shot attempt verdict: gates on candidate -> interleaved A/B vs a +# pristine-rebuilt baseline -> ledger row. Extra flags pass through +# (--patch-file, --needs-human-review, --baseline-bin). +verdict target candidate run_id class hypothesis hotspot *flags: + cargo run -p verdict -- {{target}} --rebuild-baseline \ + --candidate-bin "{{candidate}}" --run-id "{{run_id}}" \ + --playbook-class {{class}} --hypothesis "{{hypothesis}}" \ + --hotspot "{{hotspot}}" {{flags}} diff --git a/results/ledger.sqlite b/results/ledger.sqlite index de577b7..90050fb 100644 Binary files a/results/ledger.sqlite and b/results/ledger.sqlite differ