bench: automated A/A + regression-injection calibration (Phase 1)

Adds 'bench-runner calibrate': N A/A sessions counted against a <5%
false-positive bar, and N sessions with a synthetic slowdown injected
as a real busy-wait inside the candidate's timed window, counted
against a >=95% detection bar (detected = ratio CI entirely below 1.0).
Bootstrap seed varies per session so resampling noise is not
correlated across the very statistic being validated. JSON evidence
includes per-session CIs, injected wait, acceptance verdict, and env
fingerprint.

First run against the real comrak workload (20+20 sessions, pinned
core): false-positive rate 0.000, detection rate 0.950 -> PASS.
Evidence: results/calibration/2026-07-06-comrak-full-calibration.json.
Closes the SPEC 3.1 acceptance-criteria gap; remaining Phase 1 gaps:
no-net containers, perf profiling, diff-test orchestration, agent IPC.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2

crates/bench-runner/src/exec.rs

@@ -15,7 +15,7 @@ pub struct MeasuredSamples {
     pub candidate_s: Vec<f64>,
 }
 
-fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
+fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<f64> {
     let start = Instant::now();
     let status = Command::new("sh")
         .arg("-c")
@@ -24,6 +24,15 @@ fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
         .stderr(Stdio::null())
         .status()
         .with_context(|| format!("failed to spawn: {shell_cmd}"))?;
+    if inject_spin_s > 0.0 {
+        // Calibration-only: a real busy-wait inside the timed window, so an
+        // injected slowdown flows through the entire measurement pipeline
+        // (SPEC §3.1 regression-injection self-test).
+        let spin_until = start.elapsed().as_secs_f64() + inject_spin_s;
+        while start.elapsed().as_secs_f64() < spin_until {
+            std::hint::spin_loop();
+        }
+    }
     let elapsed = start.elapsed().as_secs_f64();
     anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}");
     Ok(elapsed)
@@ -31,23 +40,26 @@ fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
 
 /// Run baseline and candidate commands on the interleaved schedule and
 /// return measured wall times per side. For A/A mode, pass the same
-/// command as both sides.
+/// command as both sides. `inject_candidate_spin_s` adds a synthetic
+/// busy-wait to candidate runs only (calibration self-tests; 0.0 in
+/// normal operation).
 pub fn run_interleaved(
     baseline_cmd: &str,
     candidate_cmd: &str,
     runs_per_side: usize,
     warmup: usize,
+    inject_candidate_spin_s: f64,
 ) -> anyhow::Result<MeasuredSamples> {
     let mut out = MeasuredSamples {
         baseline_s: Vec::with_capacity(runs_per_side),
         candidate_s: Vec::with_capacity(runs_per_side),
     };
     for (side, measured) in interleaved(runs_per_side, warmup) {
-        let cmd = match side {
-            Side::Baseline => baseline_cmd,
-            Side::Candidate => candidate_cmd,
+        let (cmd, spin) = match side {
+            Side::Baseline => (baseline_cmd, 0.0),
+            Side::Candidate => (candidate_cmd, inject_candidate_spin_s),
         };
-        let elapsed = time_once(cmd)?;
+        let elapsed = time_once(cmd, spin)?;
         if measured {
             match side {
                 Side::Baseline => out.baseline_s.push(elapsed),

crates/bench-runner/src/main.rs

@@ -28,6 +28,95 @@ enum Cmd {
         #[arg(long)]
         cmd: String,
     },
+    /// Automated calibration (SPEC §3.1): N A/A sessions must show a
+    /// false-positive rate <5%, and N sessions with a synthetic
+    /// slowdown injected into the candidate must detect it ≥95% of the
+    /// time. Writes a JSON evidence file.
+    Calibrate {
+        #[arg(long)]
+        cmd: String,
+        #[arg(long, default_value_t = 20)]
+        sessions: usize,
+        /// Injected relative slowdown for the detection test.
+        #[arg(long, default_value_t = 0.05)]
+        slowdown: f64,
+        /// Output JSON evidence path.
+        #[arg(long)]
+        out: PathBuf,
+    },
+}
+
+fn calibrate(
+    cfg: &bench_runner::config::AcceptConfig,
+    cmd: &str,
+    sessions: usize,
+    slowdown: f64,
+    out: &PathBuf,
+) -> Result<()> {
+    let fp = EnvFingerprint::collect("caller-provided (wrap cmd in taskset)", "system-default");
+    // Baseline median to size the injected busy-wait.
+    let probe = exec::run_interleaved(cmd, cmd, 5, 1, 0.0)?;
+    let inject_s = stats::median(&probe.baseline_s) * slowdown;
+
+    let mut aa_sessions = Vec::new();
+    let mut inj_sessions = Vec::new();
+    let (mut false_positives, mut detections) = (0usize, 0usize);
+    for i in 0..sessions {
+        // Vary the bootstrap seed per session; resampling with one fixed
+        // seed across sessions would correlate the very statistic we are
+        // trying to validate.
+        let seed = cfg.bootstrap_seed.wrapping_add(i as u64);
+        let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
+        let aa_ci = stats::bootstrap_ratio_ci(
+            &aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
+        );
+        if decide(aa_ci, cfg.threshold) == Verdict::Accepted {
+            false_positives += 1;
+        }
+        aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
+
+        let inj =
+            exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
+        let inj_ci = stats::bootstrap_ratio_ci(
+            &inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
+        );
+        // Detected = the harness resolves a regression: CI entirely < 1.
+        if inj_ci.hi < 1.0 {
+            detections += 1;
+        }
+        inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
+        eprintln!(
+            "session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}",
+            i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi
+        );
+    }
+
+    let fp_rate = false_positives as f64 / sessions as f64;
+    let det_rate = detections as f64 / sessions as f64;
+    let pass = fp_rate < 0.05 && det_rate >= 0.95;
+    let evidence = serde_json::json!({
+        "workload_cmd": cmd,
+        "sessions": sessions,
+        "runs_per_side": cfg.runs_per_side,
+        "threshold": cfg.threshold,
+        "injected_slowdown": slowdown,
+        "injected_busy_wait_s": inject_s,
+        "aa_false_positive_rate": fp_rate,
+        "aa_sessions": aa_sessions,
+        "injection_detection_rate": det_rate,
+        "injection_sessions": inj_sessions,
+        "acceptance": {"fp_lt": 0.05, "detection_ge": 0.95, "pass": pass},
+        "env_fingerprint": fp,
+    });
+    std::fs::write(out, serde_json::to_string_pretty(&evidence)?)?;
+    println!(
+        "calibration: A/A false-positive rate {fp_rate:.3} (<0.05 required), \
+         injected-{:.0}%-slowdown detection rate {det_rate:.3} (>=0.95 required) -> {}",
+        slowdown * 100.0,
+        if pass { "PASS" } else { "FAIL" }
+    );
+    anyhow::ensure!(pass, "calibration acceptance criteria not met");
+    Ok(())
 }
 
 fn main() -> Result<()> {
@@ -40,6 +129,9 @@ fn main() -> Result<()> {
     let (baseline, candidate, aa_mode) = match &cli.command {
         Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
         Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
+        Cmd::Calibrate { cmd, sessions, slowdown, out } => {
+            return calibrate(&cfg, cmd, *sessions, *slowdown, out);
+        }
     };
 
     let fp = EnvFingerprint::collect("none", "system-default");
@@ -50,7 +142,7 @@ fn main() -> Result<()> {
     );
 
     let samples =
-        exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs)?;
+        exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
     let ci = stats::bootstrap_ratio_ci(
         &samples.baseline_s,
         &samples.candidate_s,