Bootstrap hotpath from engineering spec v0.1

Commit the spec and scaffold the system it describes:

- SPEC.md: the full v0.1 engineering spec
- crates/: trust-layer Rust workspace, unit-tested
  - bench-runner: interleaved ABAB scheduling with discarded warm-ups,
    deterministic bootstrap CIs over the ratio of medians,
    CI-lower-bound accept rule, env fingerprinting, and a CLI with
    compare + A/A self-test modes (A/A verified passing end-to-end)
  - diff-test: gate outcome model, sha256 corpus pin verifier that
    refuses on mismatch, per-target equivalence policy parser
    (byte-identical default, explicit fp-tolerance opt-in)
  - ledger: append-only SQLite attempt schema with UPDATE/DELETE
    refused by triggers, anti-double-attempt query
  - report: cores-saved and dollars/yr ROI math over speedup CIs,
    pricing.toml inputs
- playbook/: classes 1-7 with preconditions, procedure, verification
  notes, and known failure modes per class
- agent/: Python package skeleton — six-tool contract and prompting
  spine, harness IPC deferred to Phase 1
- config/: accept.toml (2% threshold, 30 runs/side), pricing.toml
- targets/, corpora/, results/calibration/: layout and policy docs

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2

crates/bench-runner/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "bench-runner"
+description = "Interleaved A/B benchmark harness with bootstrap CIs, A/A calibration, and environment fingerprinting"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+clap.workspace = true
+ledger = { path = "../ledger" }
+serde.workspace = true
+serde_json.workspace = true
+toml.workspace = true

crates/bench-runner/src/config.rs

@@ -0,0 +1,48 @@
+//! Accept-threshold configuration, loaded from `config/accept.toml`.
+
+use serde::Deserialize;
+use std::path::Path;
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct AcceptConfig {
+    /// CI lower bound must clear 1 + threshold (default 2%).
+    pub threshold: f64,
+    /// Measured runs per side (SPEC minimum: 30).
+    pub runs_per_side: usize,
+    /// Unmeasured warm-up runs per side, discarded.
+    pub warmup_runs: usize,
+    /// Bootstrap resample count.
+    pub bootstrap_iters: usize,
+    /// Confidence level for the ratio CI, e.g. 0.95.
+    pub confidence: f64,
+    /// PRNG seed for the bootstrap (reproducible calibration).
+    pub bootstrap_seed: u64,
+}
+
+impl Default for AcceptConfig {
+    fn default() -> Self {
+        Self {
+            threshold: 0.02,
+            runs_per_side: 30,
+            warmup_runs: 3,
+            bootstrap_iters: 10_000,
+            confidence: 0.95,
+            bootstrap_seed: 0x707E_17E5,
+        }
+    }
+}
+
+impl AcceptConfig {
+    pub fn load(path: &Path) -> anyhow::Result<Self> {
+        let raw = std::fs::read_to_string(path)?;
+        let cfg: Self = toml::from_str(&raw)?;
+        anyhow::ensure!(cfg.runs_per_side >= 2, "runs_per_side must be >= 2");
+        anyhow::ensure!(cfg.threshold > 0.0, "threshold must be positive");
+        anyhow::ensure!(
+            (0.0..1.0).contains(&cfg.confidence),
+            "confidence must be in (0, 1)"
+        );
+        Ok(cfg)
+    }
+}

crates/bench-runner/src/exec.rs

@@ -0,0 +1,59 @@
+//! Command execution and timing. Whole-program mode (hyperfine-style):
+//! spawn the command, measure wall time, discard warm-ups per the
+//! schedule. `perf stat` counters and RAPL energy are Phase 1 follow-ups
+//! (tracked in README roadmap); wall time alone is enough to bring up
+//! A/A calibration.
+
+use crate::schedule::{interleaved, Side};
+use anyhow::Context;
+use std::process::{Command, Stdio};
+use std::time::Instant;
+
+#[derive(Debug, Clone)]
+pub struct MeasuredSamples {
+    pub baseline_s: Vec<f64>,
+    pub candidate_s: Vec<f64>,
+}
+
+fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
+    let start = Instant::now();
+    let status = Command::new("sh")
+        .arg("-c")
+        .arg(shell_cmd)
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .with_context(|| format!("failed to spawn: {shell_cmd}"))?;
+    let elapsed = start.elapsed().as_secs_f64();
+    anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}");
+    Ok(elapsed)
+}
+
+/// Run baseline and candidate commands on the interleaved schedule and
+/// return measured wall times per side. For A/A mode, pass the same
+/// command as both sides.
+pub fn run_interleaved(
+    baseline_cmd: &str,
+    candidate_cmd: &str,
+    runs_per_side: usize,
+    warmup: usize,
+) -> anyhow::Result<MeasuredSamples> {
+    let mut out = MeasuredSamples {
+        baseline_s: Vec::with_capacity(runs_per_side),
+        candidate_s: Vec::with_capacity(runs_per_side),
+    };
+    for (side, measured) in interleaved(runs_per_side, warmup) {
+        let cmd = match side {
+            Side::Baseline => baseline_cmd,
+            Side::Candidate => candidate_cmd,
+        };
+        let elapsed = time_once(cmd)?;
+        if measured {
+            match side {
+                Side::Baseline => out.baseline_s.push(elapsed),
+                Side::Candidate => out.candidate_s.push(elapsed),
+            }
+        }
+    }
+    Ok(out)
+}

crates/bench-runner/src/fingerprint.rs

@@ -0,0 +1,53 @@
+//! Environment fingerprint (SPEC §3.1, §7): every reported number carries
+//! the machine state it was measured under. Collection is best-effort —
+//! a field we cannot read is recorded as "unknown", never omitted.
+
+use serde::Serialize;
+use std::fs;
+
+#[derive(Debug, Clone, Serialize)]
+pub struct EnvFingerprint {
+    pub kernel: String,
+    pub cpu_model: String,
+    pub governor: String,
+    pub smt: String,
+    pub turbo: String,
+    /// ASLR handling mode actually in effect for this session (SPEC §7:
+    /// randomize-and-aggregate vs. fixed; document, don't assume).
+    pub aslr: String,
+    pub cpu_pinning: String,
+}
+
+fn read_trimmed(path: &str) -> String {
+    fs::read_to_string(path)
+        .map(|s| s.trim().to_string())
+        .unwrap_or_else(|_| "unknown".to_string())
+}
+
+impl EnvFingerprint {
+    pub fn collect(pinning: &str, aslr_mode: &str) -> Self {
+        let cpu_model = fs::read_to_string("/proc/cpuinfo")
+            .ok()
+            .and_then(|s| {
+                s.lines()
+                    .find(|l| l.starts_with("model name"))
+                    .and_then(|l| l.split(':').nth(1))
+                    .map(|v| v.trim().to_string())
+            })
+            .unwrap_or_else(|| "unknown".to_string());
+        Self {
+            kernel: read_trimmed("/proc/sys/kernel/osrelease"),
+            cpu_model,
+            governor: read_trimmed(
+                "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
+            ),
+            smt: read_trimmed("/sys/devices/system/cpu/smt/active"),
+            turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"),
+            aslr: format!(
+                "mode={aslr_mode} randomize_va_space={}",
+                read_trimmed("/proc/sys/kernel/randomize_va_space")
+            ),
+            cpu_pinning: pinning.to_string(),
+        }
+    }
+}

crates/bench-runner/src/lib.rs

@@ -0,0 +1,43 @@
+//! bench-runner core (SPEC §3.1, §7).
+//!
+//! The product is this crate: interleaved A/B measurement, bootstrap CIs
+//! over the ratio of medians, threshold-gated accept decisions, and A/A
+//! self-calibration. Pure statistics live here and are unit-tested;
+//! process execution and `perf stat` integration live in `exec`.
+
+pub mod config;
+pub mod exec;
+pub mod fingerprint;
+pub mod schedule;
+pub mod stats;
+
+use ledger::Verdict;
+
+/// Decide a verdict from a speedup CI and the accept threshold.
+///
+/// Speedup is baseline/candidate (>1 means the candidate is faster).
+/// Accepted only if the CI *lower bound* clears `1 + threshold` — the
+/// point estimate is never enough (SPEC §3.1).
+pub fn decide(speedup_ci: stats::RatioCi, threshold: f64) -> Verdict {
+    if speedup_ci.lo >= 1.0 + threshold {
+        Verdict::Accepted
+    } else {
+        Verdict::RejectedBench
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::stats::RatioCi;
+
+    #[test]
+    fn accept_requires_ci_lower_bound_not_point_estimate() {
+        // 10% median speedup but the CI dips below threshold: rejected.
+        let noisy = RatioCi { median: 1.10, lo: 1.005, hi: 1.21 };
+        assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench);
+        // 4% speedup with a tight CI: accepted.
+        let tight = RatioCi { median: 1.04, lo: 1.025, hi: 1.055 };
+        assert_eq!(decide(tight, 0.02), Verdict::Accepted);
+    }
+}

crates/bench-runner/src/main.rs

@@ -0,0 +1,81 @@
+use anyhow::Result;
+use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
+use clap::{Parser, Subcommand};
+use ledger::Verdict;
+use std::path::PathBuf;
+
+#[derive(Parser)]
+#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")]
+struct Cli {
+    /// Path to accept.toml; built-in defaults if omitted.
+    #[arg(long)]
+    config: Option<PathBuf>,
+    #[command(subcommand)]
+    command: Cmd,
+}
+
+#[derive(Subcommand)]
+enum Cmd {
+    /// Interleaved A/B comparison of two shell commands.
+    Compare {
+        #[arg(long)]
+        baseline: String,
+        #[arg(long)]
+        candidate: String,
+    },
+    /// A/A self-test: same command both sides; must yield a null verdict.
+    Aa {
+        #[arg(long)]
+        cmd: String,
+    },
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let cfg = match &cli.config {
+        Some(p) => AcceptConfig::load(p)?,
+        None => AcceptConfig::default(),
+    };
+
+    let (baseline, candidate, aa_mode) = match &cli.command {
+        Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
+        Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
+    };
+
+    let fp = EnvFingerprint::collect("none", "system-default");
+    eprintln!("env fingerprint: {}", serde_json::to_string(&fp)?);
+    eprintln!(
+        "plan: {} measured + {} warm-up runs per side, interleaved",
+        cfg.runs_per_side, cfg.warmup_runs
+    );
+
+    let samples =
+        exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs)?;
+    let ci = stats::bootstrap_ratio_ci(
+        &samples.baseline_s,
+        &samples.candidate_s,
+        cfg.bootstrap_iters,
+        cfg.confidence,
+        cfg.bootstrap_seed,
+    );
+
+    println!(
+        "speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}]",
+        ci.median,
+        cfg.confidence * 100.0,
+        ci.lo,
+        ci.hi
+    );
+
+    if aa_mode {
+        // Null verdict required: an "accept" here is a calibration failure.
+        let verdict = decide(ci, cfg.threshold);
+        if verdict == Verdict::Accepted {
+            anyhow::bail!("A/A self-test FAILED: harness claims a speedup from identical binaries");
+        }
+        println!("A/A self-test passed (null verdict)");
+    } else {
+        println!("verdict: {}", decide(ci, cfg.threshold).as_str());
+    }
+    Ok(())
+}

crates/bench-runner/src/schedule.rs

@@ -0,0 +1,44 @@
+//! Interleaved run scheduling (SPEC §3.1): baseline and candidate run
+//! ABABAB…, never sequential blocks, so thermal and background drift
+//! land on both sides equally.
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Side {
+    Baseline,
+    Candidate,
+}
+
+/// ABAB… schedule with `runs_per_side` measured runs on each side,
+/// preceded by `warmup` unmeasured warm-up entries (alternating too, so
+/// both binaries reach steady state).
+pub fn interleaved(runs_per_side: usize, warmup: usize) -> Vec<(Side, bool)> {
+    let mut plan = Vec::with_capacity(2 * (runs_per_side + warmup));
+    for i in 0..(runs_per_side + warmup) {
+        let measured = i >= warmup;
+        plan.push((Side::Baseline, measured));
+        plan.push((Side::Candidate, measured));
+    }
+    plan
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn strictly_alternating_and_counts_match() {
+        let plan = interleaved(30, 3);
+        assert_eq!(plan.len(), 66);
+        for pair in plan.chunks(2) {
+            assert_eq!(pair[0].0, Side::Baseline);
+            assert_eq!(pair[1].0, Side::Candidate);
+        }
+        let measured_baseline = plan
+            .iter()
+            .filter(|(s, m)| *s == Side::Baseline && *m)
+            .count();
+        assert_eq!(measured_baseline, 30);
+        assert!(!plan[0].1 && !plan[5].1, "warm-ups are unmeasured");
+        assert!(plan[6].1, "runs after warm-up are measured");
+    }
+}

crates/bench-runner/src/stats.rs

@@ -0,0 +1,118 @@
+//! Ratios with bootstrap CIs; never means without spread; never a single
+//! run, ever (SPEC §7).
+
+/// Bootstrap confidence interval over the ratio of medians
+/// (baseline / candidate).
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct RatioCi {
+    pub median: f64,
+    pub lo: f64,
+    pub hi: f64,
+}
+
+pub fn median(xs: &[f64]) -> f64 {
+    assert!(!xs.is_empty(), "median of empty sample");
+    let mut v = xs.to_vec();
+    v.sort_by(|a, b| a.partial_cmp(b).expect("NaN in sample"));
+    let n = v.len();
+    if n % 2 == 1 {
+        v[n / 2]
+    } else {
+        (v[n / 2 - 1] + v[n / 2]) / 2.0
+    }
+}
+
+/// Deterministic xorshift64* PRNG. Seeded explicitly so calibration runs
+/// are reproducible; no wall-clock entropy anywhere in the stats path.
+struct XorShift64(u64);
+
+impl XorShift64 {
+    fn new(seed: u64) -> Self {
+        Self(seed.max(1))
+    }
+    fn next_u64(&mut self) -> u64 {
+        let mut x = self.0;
+        x ^= x >> 12;
+        x ^= x << 25;
+        x ^= x >> 27;
+        self.0 = x;
+        x.wrapping_mul(0x2545F4914F6CDD1D)
+    }
+    fn index(&mut self, n: usize) -> usize {
+        (self.next_u64() % n as u64) as usize
+    }
+}
+
+/// Percentile-method bootstrap CI of median(baseline)/median(candidate).
+///
+/// `confidence` is e.g. 0.95. Both samples are resampled independently
+/// with replacement `iters` times.
+pub fn bootstrap_ratio_ci(
+    baseline: &[f64],
+    candidate: &[f64],
+    iters: usize,
+    confidence: f64,
+    seed: u64,
+) -> RatioCi {
+    assert!(baseline.len() >= 2 && candidate.len() >= 2, "need >=2 runs per side");
+    assert!((0.0..1.0).contains(&confidence));
+    let mut rng = XorShift64::new(seed);
+    let mut ratios = Vec::with_capacity(iters);
+    let mut b_resample = vec![0.0; baseline.len()];
+    let mut c_resample = vec![0.0; candidate.len()];
+    for _ in 0..iters {
+        for slot in b_resample.iter_mut() {
+            *slot = baseline[rng.index(baseline.len())];
+        }
+        for slot in c_resample.iter_mut() {
+            *slot = candidate[rng.index(candidate.len())];
+        }
+        ratios.push(median(&b_resample) / median(&c_resample));
+    }
+    ratios.sort_by(|a, b| a.partial_cmp(b).expect("NaN ratio"));
+    let alpha = (1.0 - confidence) / 2.0;
+    let lo_idx = ((iters as f64) * alpha).floor() as usize;
+    let hi_idx = (((iters as f64) * (1.0 - alpha)).ceil() as usize).min(iters - 1);
+    RatioCi {
+        median: median(baseline) / median(candidate),
+        lo: ratios[lo_idx],
+        hi: ratios[hi_idx],
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn median_odd_and_even() {
+        assert_eq!(median(&[3.0, 1.0, 2.0]), 2.0);
+        assert_eq!(median(&[4.0, 1.0, 2.0, 3.0]), 2.5);
+    }
+
+    #[test]
+    fn aa_ci_straddles_one() {
+        // Same distribution both sides: the CI must contain 1.0.
+        let sample: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
+        let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42);
+        assert!(ci.lo <= 1.0 && ci.hi >= 1.0, "A/A CI must straddle 1.0, got {ci:?}");
+    }
+
+    #[test]
+    fn detects_injected_regression() {
+        // Candidate 5% slower with mild noise: speedup CI entirely < 1.
+        let base: Vec<f64> = (0..40).map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64).collect();
+        let cand: Vec<f64> = base.iter().map(|t| t * 1.05).collect();
+        let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42);
+        assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}");
+    }
+
+    #[test]
+    fn deterministic_for_fixed_seed() {
+        let a = vec![1.0, 1.1, 1.05, 0.98, 1.02, 1.07];
+        let b = vec![0.9, 0.95, 0.93, 0.91, 0.97, 0.92];
+        let x = bootstrap_ratio_ci(&a, &b, 500, 0.95, 7);
+        let y = bootstrap_ratio_ci(&a, &b, 500, 0.95, 7);
+        assert_eq!(x, y);
+    }
+}