gates: verdict pipeline — gates + bench + ledger row, pristine-rebuilt baselines

Adds crates/verdict ('just verdict'): runs the core gates on the
candidate binary, then an interleaved A/B bench vs the baseline with
the CI-lower-bound rule, and writes the append-only ledger row with
per-side median CIs, workload statement, and env fingerprint. Gate
failure short-circuits to rejected-gate; --needs-human-review caps an
accepted verdict (never upgrades). CPU pinning moves into accept.toml
(pin_prefix, machine-specific, applied to both sides identically).

The smoke test caught a real trust gap: pointing --baseline-bin at the
workspace build (phase1-verdict-smoke-001) measured patched-vs-patched
and returned null, because gate runs rebuild the workspace binary from
the patched tree. Fix: --rebuild-baseline stashes the patch, builds
the baseline into isolated targets/<name>/baseline/, and restores the
patch — SPEC 3.1's pristine-checkout rule, now enforced by the
harness. Re-run (phase1-verdict-smoke-002) reproduces the Phase 0
result mechanically: +10.4% median, 95% CI [1.0932, 1.1137], accepted.
Both smoke rows are in the ledger, including the null one.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
Claude
2026-07-06 23:50:05 +00:00
parent 0d29d51f72
commit baa6c4e967
15 changed files with 394 additions and 62 deletions
+1
View File
@@ -7,6 +7,7 @@
//! per-target equivalence policy parser (byte-identical unless a target
//! declares an explicit FP tolerance).
pub mod orchestrate;
pub mod pin;
pub mod target;
pub mod policy;
+9 -62
View File
@@ -1,15 +1,13 @@
//! Gate orchestration: corpus pin → upstream tests → golden replay.
//! Refuses to run anything if the corpus manifest mismatches (SPEC §3.2).
//! Gate orchestration CLI: corpus pin → upstream tests → golden replay.
//! Differential fuzz and sanitizers remain per-attempt manual steps in
//! Phase 1 and are reported as skipped with a reason.
use anyhow::Result;
use clap::Parser;
use diff_test::target::{expected_golden_hash, TargetSpec};
use diff_test::{all_passed, pin, GateLayer, GateOutcome};
use sha2::{Digest, Sha256};
use diff_test::orchestrate::run_core_gates;
use diff_test::target::TargetSpec;
use diff_test::{all_passed, GateOutcome};
use std::path::PathBuf;
use std::process::{Command, Stdio};
#[derive(Parser)]
#[command(name = "diff-test", about = "hotpath equivalence-gate orchestrator")]
@@ -21,67 +19,16 @@ struct Cli {
root: PathBuf,
}
fn sh(root: &PathBuf, cmd: &str) -> Result<std::process::Output> {
Ok(Command::new("sh")
.arg("-c")
.arg(cmd)
.current_dir(root)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()?)
}
fn main() -> Result<()> {
let cli = Cli::parse();
let root = cli.root.canonicalize()?;
let spec = TargetSpec::load(&root, &cli.target)?;
let mut results: Vec<(GateLayer, GateOutcome)> = Vec::new();
// Corpus pin is a precondition, not a gate: mismatch = refuse to run.
let verified = pin::verify_manifest(&root.join(&spec.corpus.manifest), &root.join(&spec.corpus.root))?;
println!("corpus pin: {} files verified against {}", verified, spec.corpus.manifest.display());
print!("upstream tests ... ");
let t = sh(&root, &spec.gates.tests)?;
let tests_outcome = if t.status.success() {
println!("PASS");
GateOutcome::Passed
} else {
println!("FAIL");
GateOutcome::Failed {
detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(),
}
};
results.push((GateLayer::UpstreamTests, tests_outcome));
print!("golden replay ... ");
let golden_cmd = spec.gates.golden.replace("{binary}", &spec.build.binary);
let g = sh(&root, &golden_cmd)?;
let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
let actual = format!("{:x}", Sha256::digest(&g.stdout));
let golden_outcome = if g.status.success() && actual == expected {
println!("PASS (stdout sha256 = pinned golden)");
GateOutcome::Passed
} else {
println!("FAIL");
GateOutcome::Failed {
detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()),
}
};
results.push((GateLayer::GoldenReplay, golden_outcome));
results.push((
GateLayer::DifferentialFuzz,
GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into() },
));
results.push((
GateLayer::Sanitizers,
GateOutcome::Skipped { reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into() },
));
let results = run_core_gates(&root, &spec, &spec.build.binary)?;
for (layer, outcome) in &results {
if let GateOutcome::Skipped { reason } = outcome {
println!("{layer:?}: skipped — {reason}");
match outcome {
GateOutcome::Passed => println!("{layer:?}: PASS"),
GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"),
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
}
}
if all_passed(&results) {
+78
View File
@@ -0,0 +1,78 @@
//! Core gate sequence, callable from the diff-test CLI and from verdict:
//! corpus pin (refuses on mismatch) → upstream tests → golden replay.
use crate::target::{expected_golden_hash, TargetSpec};
use crate::{pin, GateLayer, GateOutcome};
use sha2::{Digest, Sha256};
use std::path::Path;
use std::process::{Command, Stdio};
fn sh(root: &Path, cmd: &str) -> anyhow::Result<std::process::Output> {
Ok(Command::new("sh")
.arg("-c")
.arg(cmd)
.current_dir(root)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()?)
}
/// Run the mechanical gates for `target`, testing `binary` (repo-root-
/// relative) in the golden replay. Corpus-pin mismatch is an error, not
/// a gate failure: nothing runs on a tampered corpus.
pub fn run_core_gates(
root: &Path,
spec: &TargetSpec,
binary: &str,
) -> anyhow::Result<Vec<(GateLayer, GateOutcome)>> {
let verified = pin::verify_manifest(
&root.join(&spec.corpus.manifest),
&root.join(&spec.corpus.root),
)?;
eprintln!(
"corpus pin: {verified} files verified against {}",
spec.corpus.manifest.display()
);
let mut results = Vec::new();
let t = sh(root, &spec.gates.tests)?;
results.push((
GateLayer::UpstreamTests,
if t.status.success() {
GateOutcome::Passed
} else {
GateOutcome::Failed {
detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(),
}
},
));
let g = sh(root, &spec.gates.golden.replace("{binary}", binary))?;
let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
let actual = format!("{:x}", Sha256::digest(&g.stdout));
results.push((
GateLayer::GoldenReplay,
if g.status.success() && actual == expected {
GateOutcome::Passed
} else {
GateOutcome::Failed {
detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()),
}
},
));
results.push((
GateLayer::DifferentialFuzz,
GateOutcome::Skipped {
reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into(),
},
));
results.push((
GateLayer::Sanitizers,
GateOutcome::Skipped {
reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into(),
},
));
Ok(results)
}
+9
View File
@@ -9,10 +9,19 @@ use std::path::{Path, PathBuf};
pub struct TargetSpec {
pub source: Source,
pub build: Build,
pub bench: Bench,
pub gates: Gates,
pub corpus: Corpus,
}
#[derive(Debug, Deserialize)]
pub struct Bench {
/// Bench command; `{binary}` substituted per side. Run from repo root.
pub command: String,
/// Human-readable workload statement, printed with every number.
pub workload: String,
}
#[derive(Debug, Deserialize)]
pub struct Source {
pub repo: String,