diff --git a/.gitignore b/.gitignore index 770bc07..6cc2103 100644 --- a/.gitignore +++ b/.gitignore @@ -233,3 +233,4 @@ targets/comrak/candidate-*/ targets/comrak/asan*/ targets/tokei/candidate-*/ targets/tokei/asan/ +!results/calibration/*.json diff --git a/CLAUDE.md b/CLAUDE.md index 62b836a..86ac6a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,7 @@ just build / test / lint # trust-layer workspace: cargo build/test/clippy+fmt just aa [cmd] # A/A self-test of bench-runner on a shell command just compare # interleaved A/B of two shell commands, bootstrap CI just pin-check # verify corpus MANIFEST.sha256 +just calibrate # automated A/A + regression-injection calibration ``` ## Definition of done — one optimization attempt diff --git a/crates/bench-runner/src/exec.rs b/crates/bench-runner/src/exec.rs index 9e536ad..a3d2100 100644 --- a/crates/bench-runner/src/exec.rs +++ b/crates/bench-runner/src/exec.rs @@ -15,7 +15,7 @@ pub struct MeasuredSamples { pub candidate_s: Vec, } -fn time_once(shell_cmd: &str) -> anyhow::Result { +fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result { let start = Instant::now(); let status = Command::new("sh") .arg("-c") @@ -24,6 +24,15 @@ fn time_once(shell_cmd: &str) -> anyhow::Result { .stderr(Stdio::null()) .status() .with_context(|| format!("failed to spawn: {shell_cmd}"))?; + if inject_spin_s > 0.0 { + // Calibration-only: a real busy-wait inside the timed window, so an + // injected slowdown flows through the entire measurement pipeline + // (SPEC §3.1 regression-injection self-test). + let spin_until = start.elapsed().as_secs_f64() + inject_spin_s; + while start.elapsed().as_secs_f64() < spin_until { + std::hint::spin_loop(); + } + } let elapsed = start.elapsed().as_secs_f64(); anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}"); Ok(elapsed) @@ -31,23 +40,26 @@ fn time_once(shell_cmd: &str) -> anyhow::Result { /// Run baseline and candidate commands on the interleaved schedule and /// return measured wall times per side. For A/A mode, pass the same -/// command as both sides. +/// command as both sides. `inject_candidate_spin_s` adds a synthetic +/// busy-wait to candidate runs only (calibration self-tests; 0.0 in +/// normal operation). pub fn run_interleaved( baseline_cmd: &str, candidate_cmd: &str, runs_per_side: usize, warmup: usize, + inject_candidate_spin_s: f64, ) -> anyhow::Result { let mut out = MeasuredSamples { baseline_s: Vec::with_capacity(runs_per_side), candidate_s: Vec::with_capacity(runs_per_side), }; for (side, measured) in interleaved(runs_per_side, warmup) { - let cmd = match side { - Side::Baseline => baseline_cmd, - Side::Candidate => candidate_cmd, + let (cmd, spin) = match side { + Side::Baseline => (baseline_cmd, 0.0), + Side::Candidate => (candidate_cmd, inject_candidate_spin_s), }; - let elapsed = time_once(cmd)?; + let elapsed = time_once(cmd, spin)?; if measured { match side { Side::Baseline => out.baseline_s.push(elapsed), diff --git a/crates/bench-runner/src/main.rs b/crates/bench-runner/src/main.rs index dae2957..1e79e89 100644 --- a/crates/bench-runner/src/main.rs +++ b/crates/bench-runner/src/main.rs @@ -28,6 +28,95 @@ enum Cmd { #[arg(long)] cmd: String, }, + /// Automated calibration (SPEC §3.1): N A/A sessions must show a + /// false-positive rate <5%, and N sessions with a synthetic + /// slowdown injected into the candidate must detect it ≥95% of the + /// time. Writes a JSON evidence file. + Calibrate { + #[arg(long)] + cmd: String, + #[arg(long, default_value_t = 20)] + sessions: usize, + /// Injected relative slowdown for the detection test. + #[arg(long, default_value_t = 0.05)] + slowdown: f64, + /// Output JSON evidence path. + #[arg(long)] + out: PathBuf, + }, +} + +fn calibrate( + cfg: &bench_runner::config::AcceptConfig, + cmd: &str, + sessions: usize, + slowdown: f64, + out: &PathBuf, +) -> Result<()> { + let fp = EnvFingerprint::collect("caller-provided (wrap cmd in taskset)", "system-default"); + // Baseline median to size the injected busy-wait. + let probe = exec::run_interleaved(cmd, cmd, 5, 1, 0.0)?; + let inject_s = stats::median(&probe.baseline_s) * slowdown; + + let mut aa_sessions = Vec::new(); + let mut inj_sessions = Vec::new(); + let (mut false_positives, mut detections) = (0usize, 0usize); + for i in 0..sessions { + // Vary the bootstrap seed per session; resampling with one fixed + // seed across sessions would correlate the very statistic we are + // trying to validate. + let seed = cfg.bootstrap_seed.wrapping_add(i as u64); + let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?; + let aa_ci = stats::bootstrap_ratio_ci( + &aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed, + ); + if decide(aa_ci, cfg.threshold) == Verdict::Accepted { + false_positives += 1; + } + aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi})); + + let inj = + exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?; + let inj_ci = stats::bootstrap_ratio_ci( + &inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed, + ); + // Detected = the harness resolves a regression: CI entirely < 1. + if inj_ci.hi < 1.0 { + detections += 1; + } + inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi})); + eprintln!( + "session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}", + i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi + ); + } + + let fp_rate = false_positives as f64 / sessions as f64; + let det_rate = detections as f64 / sessions as f64; + let pass = fp_rate < 0.05 && det_rate >= 0.95; + let evidence = serde_json::json!({ + "workload_cmd": cmd, + "sessions": sessions, + "runs_per_side": cfg.runs_per_side, + "threshold": cfg.threshold, + "injected_slowdown": slowdown, + "injected_busy_wait_s": inject_s, + "aa_false_positive_rate": fp_rate, + "aa_sessions": aa_sessions, + "injection_detection_rate": det_rate, + "injection_sessions": inj_sessions, + "acceptance": {"fp_lt": 0.05, "detection_ge": 0.95, "pass": pass}, + "env_fingerprint": fp, + }); + std::fs::write(out, serde_json::to_string_pretty(&evidence)?)?; + println!( + "calibration: A/A false-positive rate {fp_rate:.3} (<0.05 required), \ + injected-{:.0}%-slowdown detection rate {det_rate:.3} (>=0.95 required) -> {}", + slowdown * 100.0, + if pass { "PASS" } else { "FAIL" } + ); + anyhow::ensure!(pass, "calibration acceptance criteria not met"); + Ok(()) } fn main() -> Result<()> { @@ -40,6 +129,9 @@ fn main() -> Result<()> { let (baseline, candidate, aa_mode) = match &cli.command { Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false), Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true), + Cmd::Calibrate { cmd, sessions, slowdown, out } => { + return calibrate(&cfg, cmd, *sessions, *slowdown, out); + } }; let fp = EnvFingerprint::collect("none", "system-default"); @@ -50,7 +142,7 @@ fn main() -> Result<()> { ); let samples = - exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs)?; + exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?; let ci = stats::bootstrap_ratio_ci( &samples.baseline_s, &samples.candidate_s, diff --git a/justfile b/justfile index 9a6a647..9453cf5 100644 --- a/justfile +++ b/justfile @@ -27,3 +27,9 @@ compare baseline candidate: # (wired through diff-test CLI once it exists; manual check for now) pin-check target: cd corpora/{{target}} && sh gen-corpus.sh + +# Automated calibration: N A/A sessions (<5% false-positive required) + +# N injected-slowdown sessions (>=95% detection required), JSON evidence +calibrate cmd out sessions="20": + cargo run -p bench-runner -- --config config/accept.toml calibrate \ + --cmd "{{cmd}}" --sessions {{sessions}} --out "{{out}}" diff --git a/results/calibration/2026-07-06-comrak-full-calibration.json b/results/calibration/2026-07-06-comrak-full-calibration.json new file mode 100644 index 0000000..e2a6c0c --- /dev/null +++ b/results/calibration/2026-07-06-comrak-full-calibration.json @@ -0,0 +1,228 @@ +{ + "aa_false_positive_rate": 0.0, + "aa_sessions": [ + { + "hi": 1.015998931989443, + "lo": 0.9905153352677495, + "median": 1.0012776949264486 + }, + { + "hi": 1.0055502756953207, + "lo": 0.981646945965943, + "median": 0.9908293785706317 + }, + { + "hi": 1.0204094554652672, + "lo": 0.9846287241295675, + "median": 1.002373764499574 + }, + { + "hi": 1.0047915300411685, + "lo": 0.9860341673780392, + "median": 0.9964988810591069 + }, + { + "hi": 1.0102726750450453, + "lo": 0.9876236508283872, + "median": 0.9947054654657848 + }, + { + "hi": 1.0298441811992136, + "lo": 0.9876526600106184, + "median": 1.0086506615393056 + }, + { + "hi": 1.0099440547974916, + "lo": 0.9822805275362518, + "median": 0.993850347123508 + }, + { + "hi": 1.00385962045072, + "lo": 0.9848165937114653, + "median": 0.9939030705425225 + }, + { + "hi": 1.0115679493048169, + "lo": 0.9909357476627676, + "median": 1.0008452273219608 + }, + { + "hi": 1.013718263355025, + "lo": 0.977768078322329, + "median": 0.9983105725894315 + }, + { + "hi": 1.0070675650128182, + "lo": 0.9868898707776302, + "median": 0.9979255224721003 + }, + { + "hi": 1.01473324312686, + "lo": 0.984779294044258, + "median": 0.9942794507746874 + }, + { + "hi": 1.0484508437256985, + "lo": 0.9663266833242238, + "median": 1.0028654978002207 + }, + { + "hi": 1.0153007892092838, + "lo": 0.9903096574162721, + "median": 1.0049001338403383 + }, + { + "hi": 1.0092343435725986, + "lo": 0.9922852873631642, + "median": 1.0034947624362798 + }, + { + "hi": 1.0125833696661481, + "lo": 0.9858629698133391, + "median": 0.9945866155256149 + }, + { + "hi": 1.0322221482398073, + "lo": 0.9951458973996147, + "median": 1.0075699560145404 + }, + { + "hi": 1.009761041226371, + "lo": 0.9898464904505755, + "median": 0.998425451880562 + }, + { + "hi": 1.016631057133635, + "lo": 0.9805588949385654, + "median": 0.997950429996102 + }, + { + "hi": 1.0200791563569052, + "lo": 0.9873713898804045, + "median": 1.000136749596359 + } + ], + "acceptance": { + "detection_ge": 0.95, + "fp_lt": 0.05, + "pass": true + }, + "env_fingerprint": { + "aslr": "mode=system-default randomize_va_space=2", + "cpu_model": "Intel(R) Xeon(R) Processor @ 2.80GHz", + "cpu_pinning": "caller-provided (wrap cmd in taskset)", + "governor": "unknown", + "kernel": "6.18.5", + "smt": "0", + "turbo": "unknown" + }, + "injected_busy_wait_s": 0.011125513, + "injected_slowdown": 0.05, + "injection_detection_rate": 0.95, + "injection_sessions": [ + { + "hi": 0.9705730747977983, + "lo": 0.9384443885194832, + "median": 0.9570129481885624 + }, + { + "hi": 0.9691260796731174, + "lo": 0.9309798719977085, + "median": 0.9411908680460361 + }, + { + "hi": 0.9775705854771548, + "lo": 0.9464129998680949, + "median": 0.9569584325473247 + }, + { + "hi": 0.9738958239067871, + "lo": 0.9490577184781656, + "median": 0.9624758687422929 + }, + { + "hi": 0.9692636780671424, + "lo": 0.9448230573540036, + "median": 0.9611951204744579 + }, + { + "hi": 0.960038069546381, + "lo": 0.9412872028306517, + "median": 0.953204396987671 + }, + { + "hi": 0.970357721583405, + "lo": 0.9312571815624449, + "median": 0.9497958159533386 + }, + { + "hi": 1.0189632738942447, + "lo": 0.8979330813534421, + "median": 0.9650177023489201 + }, + { + "hi": 0.9715620167742915, + "lo": 0.9336764011030461, + "median": 0.9512964417642272 + }, + { + "hi": 0.9768953182714468, + "lo": 0.9385015313970146, + "median": 0.9538566142021077 + }, + { + "hi": 0.9733246543232819, + "lo": 0.9452506173210715, + "median": 0.9558422793746748 + }, + { + "hi": 0.968562587511655, + "lo": 0.9364861001420586, + "median": 0.9461689697819208 + }, + { + "hi": 0.9832173753801465, + "lo": 0.9352025813901989, + "median": 0.9618202496291594 + }, + { + "hi": 0.9701329322900167, + "lo": 0.9395861281724339, + "median": 0.954612721677948 + }, + { + "hi": 0.9921703664327655, + "lo": 0.9357083455043089, + "median": 0.958758177852707 + }, + { + "hi": 0.976082080482069, + "lo": 0.9328145868694007, + "median": 0.947453955805073 + }, + { + "hi": 0.9795318432851117, + "lo": 0.9317744456440425, + "median": 0.9607784721426458 + }, + { + "hi": 0.9751440256938523, + "lo": 0.9491637711903045, + "median": 0.9646173778660493 + }, + { + "hi": 0.9730959153169779, + "lo": 0.928638531103063, + "median": 0.9490375911197446 + }, + { + "hi": 0.9630989374168489, + "lo": 0.9484798599580982, + "median": 0.9548094809061651 + } + ], + "runs_per_side": 30, + "sessions": 20, + "threshold": 0.02, + "workload_cmd": "taskset -c 2 sh -c 'targets/comrak/workspace/target/release/comrak < corpora/comrak/progit.md > /dev/null'" +} \ No newline at end of file