mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
bench: automated A/A + regression-injection calibration (Phase 1)
Adds 'bench-runner calibrate': N A/A sessions counted against a <5% false-positive bar, and N sessions with a synthetic slowdown injected as a real busy-wait inside the candidate's timed window, counted against a >=95% detection bar (detected = ratio CI entirely below 1.0). Bootstrap seed varies per session so resampling noise is not correlated across the very statistic being validated. JSON evidence includes per-session CIs, injected wait, acceptance verdict, and env fingerprint. First run against the real comrak workload (20+20 sessions, pinned core): false-positive rate 0.000, detection rate 0.950 -> PASS. Evidence: results/calibration/2026-07-06-comrak-full-calibration.json. Closes the SPEC 3.1 acceptance-criteria gap; remaining Phase 1 gaps: no-net containers, perf profiling, diff-test orchestration, agent IPC. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
Claude
@@ -0,0 +1,228 @@
|
||||
{
|
||||
"aa_false_positive_rate": 0.0,
|
||||
"aa_sessions": [
|
||||
{
|
||||
"hi": 1.015998931989443,
|
||||
"lo": 0.9905153352677495,
|
||||
"median": 1.0012776949264486
|
||||
},
|
||||
{
|
||||
"hi": 1.0055502756953207,
|
||||
"lo": 0.981646945965943,
|
||||
"median": 0.9908293785706317
|
||||
},
|
||||
{
|
||||
"hi": 1.0204094554652672,
|
||||
"lo": 0.9846287241295675,
|
||||
"median": 1.002373764499574
|
||||
},
|
||||
{
|
||||
"hi": 1.0047915300411685,
|
||||
"lo": 0.9860341673780392,
|
||||
"median": 0.9964988810591069
|
||||
},
|
||||
{
|
||||
"hi": 1.0102726750450453,
|
||||
"lo": 0.9876236508283872,
|
||||
"median": 0.9947054654657848
|
||||
},
|
||||
{
|
||||
"hi": 1.0298441811992136,
|
||||
"lo": 0.9876526600106184,
|
||||
"median": 1.0086506615393056
|
||||
},
|
||||
{
|
||||
"hi": 1.0099440547974916,
|
||||
"lo": 0.9822805275362518,
|
||||
"median": 0.993850347123508
|
||||
},
|
||||
{
|
||||
"hi": 1.00385962045072,
|
||||
"lo": 0.9848165937114653,
|
||||
"median": 0.9939030705425225
|
||||
},
|
||||
{
|
||||
"hi": 1.0115679493048169,
|
||||
"lo": 0.9909357476627676,
|
||||
"median": 1.0008452273219608
|
||||
},
|
||||
{
|
||||
"hi": 1.013718263355025,
|
||||
"lo": 0.977768078322329,
|
||||
"median": 0.9983105725894315
|
||||
},
|
||||
{
|
||||
"hi": 1.0070675650128182,
|
||||
"lo": 0.9868898707776302,
|
||||
"median": 0.9979255224721003
|
||||
},
|
||||
{
|
||||
"hi": 1.01473324312686,
|
||||
"lo": 0.984779294044258,
|
||||
"median": 0.9942794507746874
|
||||
},
|
||||
{
|
||||
"hi": 1.0484508437256985,
|
||||
"lo": 0.9663266833242238,
|
||||
"median": 1.0028654978002207
|
||||
},
|
||||
{
|
||||
"hi": 1.0153007892092838,
|
||||
"lo": 0.9903096574162721,
|
||||
"median": 1.0049001338403383
|
||||
},
|
||||
{
|
||||
"hi": 1.0092343435725986,
|
||||
"lo": 0.9922852873631642,
|
||||
"median": 1.0034947624362798
|
||||
},
|
||||
{
|
||||
"hi": 1.0125833696661481,
|
||||
"lo": 0.9858629698133391,
|
||||
"median": 0.9945866155256149
|
||||
},
|
||||
{
|
||||
"hi": 1.0322221482398073,
|
||||
"lo": 0.9951458973996147,
|
||||
"median": 1.0075699560145404
|
||||
},
|
||||
{
|
||||
"hi": 1.009761041226371,
|
||||
"lo": 0.9898464904505755,
|
||||
"median": 0.998425451880562
|
||||
},
|
||||
{
|
||||
"hi": 1.016631057133635,
|
||||
"lo": 0.9805588949385654,
|
||||
"median": 0.997950429996102
|
||||
},
|
||||
{
|
||||
"hi": 1.0200791563569052,
|
||||
"lo": 0.9873713898804045,
|
||||
"median": 1.000136749596359
|
||||
}
|
||||
],
|
||||
"acceptance": {
|
||||
"detection_ge": 0.95,
|
||||
"fp_lt": 0.05,
|
||||
"pass": true
|
||||
},
|
||||
"env_fingerprint": {
|
||||
"aslr": "mode=system-default randomize_va_space=2",
|
||||
"cpu_model": "Intel(R) Xeon(R) Processor @ 2.80GHz",
|
||||
"cpu_pinning": "caller-provided (wrap cmd in taskset)",
|
||||
"governor": "unknown",
|
||||
"kernel": "6.18.5",
|
||||
"smt": "0",
|
||||
"turbo": "unknown"
|
||||
},
|
||||
"injected_busy_wait_s": 0.011125513,
|
||||
"injected_slowdown": 0.05,
|
||||
"injection_detection_rate": 0.95,
|
||||
"injection_sessions": [
|
||||
{
|
||||
"hi": 0.9705730747977983,
|
||||
"lo": 0.9384443885194832,
|
||||
"median": 0.9570129481885624
|
||||
},
|
||||
{
|
||||
"hi": 0.9691260796731174,
|
||||
"lo": 0.9309798719977085,
|
||||
"median": 0.9411908680460361
|
||||
},
|
||||
{
|
||||
"hi": 0.9775705854771548,
|
||||
"lo": 0.9464129998680949,
|
||||
"median": 0.9569584325473247
|
||||
},
|
||||
{
|
||||
"hi": 0.9738958239067871,
|
||||
"lo": 0.9490577184781656,
|
||||
"median": 0.9624758687422929
|
||||
},
|
||||
{
|
||||
"hi": 0.9692636780671424,
|
||||
"lo": 0.9448230573540036,
|
||||
"median": 0.9611951204744579
|
||||
},
|
||||
{
|
||||
"hi": 0.960038069546381,
|
||||
"lo": 0.9412872028306517,
|
||||
"median": 0.953204396987671
|
||||
},
|
||||
{
|
||||
"hi": 0.970357721583405,
|
||||
"lo": 0.9312571815624449,
|
||||
"median": 0.9497958159533386
|
||||
},
|
||||
{
|
||||
"hi": 1.0189632738942447,
|
||||
"lo": 0.8979330813534421,
|
||||
"median": 0.9650177023489201
|
||||
},
|
||||
{
|
||||
"hi": 0.9715620167742915,
|
||||
"lo": 0.9336764011030461,
|
||||
"median": 0.9512964417642272
|
||||
},
|
||||
{
|
||||
"hi": 0.9768953182714468,
|
||||
"lo": 0.9385015313970146,
|
||||
"median": 0.9538566142021077
|
||||
},
|
||||
{
|
||||
"hi": 0.9733246543232819,
|
||||
"lo": 0.9452506173210715,
|
||||
"median": 0.9558422793746748
|
||||
},
|
||||
{
|
||||
"hi": 0.968562587511655,
|
||||
"lo": 0.9364861001420586,
|
||||
"median": 0.9461689697819208
|
||||
},
|
||||
{
|
||||
"hi": 0.9832173753801465,
|
||||
"lo": 0.9352025813901989,
|
||||
"median": 0.9618202496291594
|
||||
},
|
||||
{
|
||||
"hi": 0.9701329322900167,
|
||||
"lo": 0.9395861281724339,
|
||||
"median": 0.954612721677948
|
||||
},
|
||||
{
|
||||
"hi": 0.9921703664327655,
|
||||
"lo": 0.9357083455043089,
|
||||
"median": 0.958758177852707
|
||||
},
|
||||
{
|
||||
"hi": 0.976082080482069,
|
||||
"lo": 0.9328145868694007,
|
||||
"median": 0.947453955805073
|
||||
},
|
||||
{
|
||||
"hi": 0.9795318432851117,
|
||||
"lo": 0.9317744456440425,
|
||||
"median": 0.9607784721426458
|
||||
},
|
||||
{
|
||||
"hi": 0.9751440256938523,
|
||||
"lo": 0.9491637711903045,
|
||||
"median": 0.9646173778660493
|
||||
},
|
||||
{
|
||||
"hi": 0.9730959153169779,
|
||||
"lo": 0.928638531103063,
|
||||
"median": 0.9490375911197446
|
||||
},
|
||||
{
|
||||
"hi": 0.9630989374168489,
|
||||
"lo": 0.9484798599580982,
|
||||
"median": 0.9548094809061651
|
||||
}
|
||||
],
|
||||
"runs_per_side": 30,
|
||||
"sessions": 20,
|
||||
"threshold": 0.02,
|
||||
"workload_cmd": "taskset -c 2 sh -c 'targets/comrak/workspace/target/release/comrak < corpora/comrak/progit.md > /dev/null'"
|
||||
}
|
||||
Reference in New Issue
Block a user