benbrown/peltier.io
2
1
Fork 0
mirror of https://github.com/bdbrown4/peltier.io.git synced 2026-11-07 10:58:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3 from bdbrown4/claude/test-wsk42i

Browse Source
This commit is contained in:
benbrown
2026-07-06 19:09:09 -04:00
committed by GitHub
parent e69d2c46ed 73763321c6
commit 7cd9db4971
6 changed files with 347 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -233,3 +233,4 @@ targets/comrak/candidate-*/
targets/comrak/asan*/
targets/tokei/candidate-*/
targets/tokei/asan/
!results/calibration/*.json
CLAUDE.md
+1
View File
@@ -54,6 +54,7 @@ just build / test / lint # trust-layer workspace: cargo build/test/clippy+fmt
just aa [cmd] # A/A self-test of bench-runner on a shell command
just compare <a> <b> # interleaved A/B of two shell commands, bootstrap CI
just pin-check <target> # verify corpus MANIFEST.sha256
just calibrate <cmd> <out> # automated A/A + regression-injection calibration
```
## Definition of done — one optimization attempt
crates/bench-runner/src/exec.rs
+18 -6
View File
@@ -15,7 +15,7 @@ pub struct MeasuredSamples {
pub candidate_s: Vec<f64>,
}
fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<f64> {
let start = Instant::now();
let status = Command::new("sh")
.arg("-c")
@@ -24,6 +24,15 @@ fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
.stderr(Stdio::null())
.status()
.with_context(|| format!("failed to spawn: {shell_cmd}"))?;
if inject_spin_s > 0.0 {
// Calibration-only: a real busy-wait inside the timed window, so an
// injected slowdown flows through the entire measurement pipeline
// (SPEC §3.1 regression-injection self-test).
let spin_until = start.elapsed().as_secs_f64() + inject_spin_s;
while start.elapsed().as_secs_f64() < spin_until {
std::hint::spin_loop();
}
}
let elapsed = start.elapsed().as_secs_f64();
anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}");
Ok(elapsed)
@@ -31,23 +40,26 @@ fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
/// Run baseline and candidate commands on the interleaved schedule and
/// return measured wall times per side. For A/A mode, pass the same
/// command as both sides.
/// command as both sides. `inject_candidate_spin_s` adds a synthetic
/// busy-wait to candidate runs only (calibration self-tests; 0.0 in
/// normal operation).
pub fn run_interleaved(
baseline_cmd: &str,
candidate_cmd: &str,
runs_per_side: usize,
warmup: usize,
inject_candidate_spin_s: f64,
) -> anyhow::Result<MeasuredSamples> {
let mut out = MeasuredSamples {
baseline_s: Vec::with_capacity(runs_per_side),
candidate_s: Vec::with_capacity(runs_per_side),
};
for (side, measured) in interleaved(runs_per_side, warmup) {
let cmd = match side {
Side::Baseline => baseline_cmd,
Side::Candidate => candidate_cmd,
let (cmd, spin) = match side {
Side::Baseline => (baseline_cmd, 0.0),
Side::Candidate => (candidate_cmd, inject_candidate_spin_s),
};
let elapsed = time_once(cmd)?;
let elapsed = time_once(cmd, spin)?;
if measured {
match side {
Side::Baseline => out.baseline_s.push(elapsed),
crates/bench-runner/src/main.rs
+93 -1
View File
@@ -28,6 +28,95 @@ enum Cmd {
#[arg(long)]
cmd: String,
},
/// Automated calibration (SPEC §3.1): N A/A sessions must show a
/// false-positive rate <5%, and N sessions with a synthetic
/// slowdown injected into the candidate must detect it ≥95% of the
/// time. Writes a JSON evidence file.
Calibrate {
#[arg(long)]
cmd: String,
#[arg(long, default_value_t = 20)]
sessions: usize,
/// Injected relative slowdown for the detection test.
#[arg(long, default_value_t = 0.05)]
slowdown: f64,
/// Output JSON evidence path.
#[arg(long)]
out: PathBuf,
},
}
fn calibrate(
cfg: &bench_runner::config::AcceptConfig,
cmd: &str,
sessions: usize,
slowdown: f64,
out: &PathBuf,
) -> Result<()> {
let fp = EnvFingerprint::collect("caller-provided (wrap cmd in taskset)", "system-default");
// Baseline median to size the injected busy-wait.
let probe = exec::run_interleaved(cmd, cmd, 5, 1, 0.0)?;
let inject_s = stats::median(&probe.baseline_s) * slowdown;
let mut aa_sessions = Vec::new();
let mut inj_sessions = Vec::new();
let (mut false_positives, mut detections) = (0usize, 0usize);
for i in 0..sessions {
// Vary the bootstrap seed per session; resampling with one fixed
// seed across sessions would correlate the very statistic we are
// trying to validate.
let seed = cfg.bootstrap_seed.wrapping_add(i as u64);
let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
let aa_ci = stats::bootstrap_ratio_ci(
&aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
);
if decide(aa_ci, cfg.threshold) == Verdict::Accepted {
false_positives += 1;
}
aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
let inj =
exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
let inj_ci = stats::bootstrap_ratio_ci(
&inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
);
// Detected = the harness resolves a regression: CI entirely < 1.
if inj_ci.hi < 1.0 {
detections += 1;
}
inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
eprintln!(
"session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}",
i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi
);
}
let fp_rate = false_positives as f64 / sessions as f64;
let det_rate = detections as f64 / sessions as f64;
let pass = fp_rate < 0.05 && det_rate >= 0.95;
let evidence = serde_json::json!({
"workload_cmd": cmd,
"sessions": sessions,
"runs_per_side": cfg.runs_per_side,
"threshold": cfg.threshold,
"injected_slowdown": slowdown,
"injected_busy_wait_s": inject_s,
"aa_false_positive_rate": fp_rate,
"aa_sessions": aa_sessions,
"injection_detection_rate": det_rate,
"injection_sessions": inj_sessions,
"acceptance": {"fp_lt": 0.05, "detection_ge": 0.95, "pass": pass},
"env_fingerprint": fp,
});
std::fs::write(out, serde_json::to_string_pretty(&evidence)?)?;
println!(
"calibration: A/A false-positive rate {fp_rate:.3} (<0.05 required), \
injected-{:.0}%-slowdown detection rate {det_rate:.3} (>=0.95 required) -> {}",
slowdown * 100.0,
if pass { "PASS" } else { "FAIL" }
);
anyhow::ensure!(pass, "calibration acceptance criteria not met");
Ok(())
}
fn main() -> Result<()> {
@@ -40,6 +129,9 @@ fn main() -> Result<()> {
let (baseline, candidate, aa_mode) = match &cli.command {
Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
Cmd::Calibrate { cmd, sessions, slowdown, out } => {
return calibrate(&cfg, cmd, *sessions, *slowdown, out);
}
};
let fp = EnvFingerprint::collect("none", "system-default");
@@ -50,7 +142,7 @@ fn main() -> Result<()> {
);
let samples =
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs)?;
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
let ci = stats::bootstrap_ratio_ci(
&samples.baseline_s,
&samples.candidate_s,
justfile
+6
View File
@@ -27,3 +27,9 @@ compare baseline candidate:
# (wired through diff-test CLI once it exists; manual check for now)
pin-check target:
cd corpora/{{target}} && sh gen-corpus.sh
# Automated calibration: N A/A sessions (<5% false-positive required) +
# N injected-slowdown sessions (>=95% detection required), JSON evidence
calibrate cmd out sessions="20":
cargo run -p bench-runner -- --config config/accept.toml calibrate \
--cmd "{{cmd}}" --sessions {{sessions}} --out "{{out}}"
results/calibration/2026-07-06-comrak-full-calibration.json
+228
View File
@@ -0,0 +1,228 @@
{
"aa_false_positive_rate": 0.0,
"aa_sessions": [
{
"hi": 1.015998931989443,
"lo": 0.9905153352677495,
"median": 1.0012776949264486
},
{
"hi": 1.0055502756953207,
"lo": 0.981646945965943,
"median": 0.9908293785706317
},
{
"hi": 1.0204094554652672,
"lo": 0.9846287241295675,
"median": 1.002373764499574
},
{
"hi": 1.0047915300411685,
"lo": 0.9860341673780392,
"median": 0.9964988810591069
},
{
"hi": 1.0102726750450453,
"lo": 0.9876236508283872,
"median": 0.9947054654657848
},
{
"hi": 1.0298441811992136,
"lo": 0.9876526600106184,
"median": 1.0086506615393056
},
{
"hi": 1.0099440547974916,
"lo": 0.9822805275362518,
"median": 0.993850347123508
},
{
"hi": 1.00385962045072,
"lo": 0.9848165937114653,
"median": 0.9939030705425225
},
{
"hi": 1.0115679493048169,
"lo": 0.9909357476627676,
"median": 1.0008452273219608
},
{
"hi": 1.013718263355025,
"lo": 0.977768078322329,
"median": 0.9983105725894315
},
{
"hi": 1.0070675650128182,
"lo": 0.9868898707776302,
"median": 0.9979255224721003
},
{
"hi": 1.01473324312686,
"lo": 0.984779294044258,
"median": 0.9942794507746874
},
{
"hi": 1.0484508437256985,
"lo": 0.9663266833242238,
"median": 1.0028654978002207
},
{
"hi": 1.0153007892092838,
"lo": 0.9903096574162721,
"median": 1.0049001338403383
},
{
"hi": 1.0092343435725986,
"lo": 0.9922852873631642,
"median": 1.0034947624362798
},
{
"hi": 1.0125833696661481,
"lo": 0.9858629698133391,
"median": 0.9945866155256149
},
{
"hi": 1.0322221482398073,
"lo": 0.9951458973996147,
"median": 1.0075699560145404
},
{
"hi": 1.009761041226371,
"lo": 0.9898464904505755,
"median": 0.998425451880562
},
{
"hi": 1.016631057133635,
"lo": 0.9805588949385654,
"median": 0.997950429996102
},
{
"hi": 1.0200791563569052,
"lo": 0.9873713898804045,
"median": 1.000136749596359
}
],
"acceptance": {
"detection_ge": 0.95,
"fp_lt": 0.05,
"pass": true
},
"env_fingerprint": {
"aslr": "mode=system-default randomize_va_space=2",
"cpu_model": "Intel(R) Xeon(R) Processor @ 2.80GHz",
"cpu_pinning": "caller-provided (wrap cmd in taskset)",
"governor": "unknown",
"kernel": "6.18.5",
"smt": "0",
"turbo": "unknown"
},
"injected_busy_wait_s": 0.011125513,
"injected_slowdown": 0.05,
"injection_detection_rate": 0.95,
"injection_sessions": [
{
"hi": 0.9705730747977983,
"lo": 0.9384443885194832,
"median": 0.9570129481885624
},
{
"hi": 0.9691260796731174,
"lo": 0.9309798719977085,
"median": 0.9411908680460361
},
{
"hi": 0.9775705854771548,
"lo": 0.9464129998680949,
"median": 0.9569584325473247
},
{
"hi": 0.9738958239067871,
"lo": 0.9490577184781656,
"median": 0.9624758687422929
},
{
"hi": 0.9692636780671424,
"lo": 0.9448230573540036,
"median": 0.9611951204744579
},
{
"hi": 0.960038069546381,
"lo": 0.9412872028306517,
"median": 0.953204396987671
},
{
"hi": 0.970357721583405,
"lo": 0.9312571815624449,
"median": 0.9497958159533386
},
{
"hi": 1.0189632738942447,
"lo": 0.8979330813534421,
"median": 0.9650177023489201
},
{
"hi": 0.9715620167742915,
"lo": 0.9336764011030461,
"median": 0.9512964417642272
},
{
"hi": 0.9768953182714468,
"lo": 0.9385015313970146,
"median": 0.9538566142021077
},
{
"hi": 0.9733246543232819,
"lo": 0.9452506173210715,
"median": 0.9558422793746748
},
{
"hi": 0.968562587511655,
"lo": 0.9364861001420586,
"median": 0.9461689697819208
},
{
"hi": 0.9832173753801465,
"lo": 0.9352025813901989,
"median": 0.9618202496291594
},
{
"hi": 0.9701329322900167,
"lo": 0.9395861281724339,
"median": 0.954612721677948
},
{
"hi": 0.9921703664327655,
"lo": 0.9357083455043089,
"median": 0.958758177852707
},
{
"hi": 0.976082080482069,
"lo": 0.9328145868694007,
"median": 0.947453955805073
},
{
"hi": 0.9795318432851117,
"lo": 0.9317744456440425,
"median": 0.9607784721426458
},
{
"hi": 0.9751440256938523,
"lo": 0.9491637711903045,
"median": 0.9646173778660493
},
{
"hi": 0.9730959153169779,
"lo": 0.928638531103063,
"median": 0.9490375911197446
},
{
"hi": 0.9630989374168489,
"lo": 0.9484798599580982,
"median": 0.9548094809061651
}
],
"runs_per_side": 30,
"sessions": 20,
"threshold": 0.02,
"workload_cmd": "taskset -c 2 sh -c 'targets/comrak/workspace/target/release/comrak < corpora/comrak/progit.md > /dev/null'"
}