mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
bench: GitHub Actions CI — trust-layer checks, no-net gates, runner calibration
Closes the CI-reachable slice of the Phase 1 infra gaps: - trust-layer job: cargo test + clippy -D warnings + fmt - target-gates-no-net: fetches pinned targets (new targets/fetch.sh), regenerates + verifies corpora, runs 'just gates tokei' mechanically, and executes the bench workload inside a --network=none container (SPEC 3.1/10 no-net requirement) - runner-calibration (informational): A/A + injection harness measures the shared runner's noise floor per run, evidence uploaded as an artifact — environments are judged empirically, not assumed - perf-probe (informational): reports which perf event sources work docs/infra.md maps each remaining hardware requirement (pinned cores, governor/SMT control, separate-uid trust layer) to the bench-metal self-hosted runner setup a human can complete in ~15 min. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
@@ -38,9 +38,7 @@ impl EnvFingerprint {
|
||||
Self {
|
||||
kernel: read_trimmed("/proc/sys/kernel/osrelease"),
|
||||
cpu_model,
|
||||
governor: read_trimmed(
|
||||
"/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
|
||||
),
|
||||
governor: read_trimmed("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"),
|
||||
smt: read_trimmed("/sys/devices/system/cpu/smt/active"),
|
||||
turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"),
|
||||
aslr: format!(
|
||||
|
||||
@@ -34,10 +34,18 @@ mod tests {
|
||||
#[test]
|
||||
fn accept_requires_ci_lower_bound_not_point_estimate() {
|
||||
// 10% median speedup but the CI dips below threshold: rejected.
|
||||
let noisy = RatioCi { median: 1.10, lo: 1.005, hi: 1.21 };
|
||||
let noisy = RatioCi {
|
||||
median: 1.10,
|
||||
lo: 1.005,
|
||||
hi: 1.21,
|
||||
};
|
||||
assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench);
|
||||
// 4% speedup with a tight CI: accepted.
|
||||
let tight = RatioCi { median: 1.04, lo: 1.025, hi: 1.055 };
|
||||
let tight = RatioCi {
|
||||
median: 1.04,
|
||||
lo: 1.025,
|
||||
hi: 1.055,
|
||||
};
|
||||
assert_eq!(decide(tight, 0.02), Verdict::Accepted);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,10 @@ use ledger::Verdict;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")]
|
||||
#[command(
|
||||
name = "bench-runner",
|
||||
about = "Interleaved A/B benchmark harness (hotpath trust layer)"
|
||||
)]
|
||||
struct Cli {
|
||||
/// Path to accept.toml; built-in defaults if omitted.
|
||||
#[arg(long)]
|
||||
@@ -68,23 +71,32 @@ fn calibrate(
|
||||
let seed = cfg.bootstrap_seed.wrapping_add(i as u64);
|
||||
let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
|
||||
let aa_ci = stats::bootstrap_ratio_ci(
|
||||
&aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
|
||||
&aa.baseline_s,
|
||||
&aa.candidate_s,
|
||||
cfg.bootstrap_iters,
|
||||
cfg.confidence,
|
||||
seed,
|
||||
);
|
||||
if decide(aa_ci, cfg.threshold) == Verdict::Accepted {
|
||||
false_positives += 1;
|
||||
}
|
||||
aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
|
||||
aa_sessions
|
||||
.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
|
||||
|
||||
let inj =
|
||||
exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
|
||||
let inj = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
|
||||
let inj_ci = stats::bootstrap_ratio_ci(
|
||||
&inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
|
||||
&inj.baseline_s,
|
||||
&inj.candidate_s,
|
||||
cfg.bootstrap_iters,
|
||||
cfg.confidence,
|
||||
seed,
|
||||
);
|
||||
// Detected = the harness resolves a regression: CI entirely < 1.
|
||||
if inj_ci.hi < 1.0 {
|
||||
detections += 1;
|
||||
}
|
||||
inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
|
||||
inj_sessions
|
||||
.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
|
||||
eprintln!(
|
||||
"session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}",
|
||||
i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi
|
||||
@@ -127,9 +139,17 @@ fn main() -> Result<()> {
|
||||
};
|
||||
|
||||
let (baseline, candidate, aa_mode) = match &cli.command {
|
||||
Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
|
||||
Cmd::Compare {
|
||||
baseline,
|
||||
candidate,
|
||||
} => (baseline.clone(), candidate.clone(), false),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
||||
Cmd::Calibrate { cmd, sessions, slowdown, out } => {
|
||||
Cmd::Calibrate {
|
||||
cmd,
|
||||
sessions,
|
||||
slowdown,
|
||||
out,
|
||||
} => {
|
||||
return calibrate(&cfg, cmd, *sessions, *slowdown, out);
|
||||
}
|
||||
};
|
||||
@@ -141,8 +161,13 @@ fn main() -> Result<()> {
|
||||
cfg.runs_per_side, cfg.warmup_runs
|
||||
);
|
||||
|
||||
let samples =
|
||||
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
|
||||
let samples = exec::run_interleaved(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
0.0,
|
||||
)?;
|
||||
let ci = stats::bootstrap_ratio_ci(
|
||||
&samples.baseline_s,
|
||||
&samples.candidate_s,
|
||||
|
||||
@@ -54,7 +54,10 @@ pub fn bootstrap_ratio_ci(
|
||||
confidence: f64,
|
||||
seed: u64,
|
||||
) -> RatioCi {
|
||||
assert!(baseline.len() >= 2 && candidate.len() >= 2, "need >=2 runs per side");
|
||||
assert!(
|
||||
baseline.len() >= 2 && candidate.len() >= 2,
|
||||
"need >=2 runs per side"
|
||||
);
|
||||
assert!((0.0..1.0).contains(&confidence));
|
||||
let mut rng = XorShift64::new(seed);
|
||||
let mut ratios = Vec::with_capacity(iters);
|
||||
@@ -82,7 +85,12 @@ pub fn bootstrap_ratio_ci(
|
||||
|
||||
/// Percentile-method bootstrap CI of a single sample's median, for
|
||||
/// per-side evidence in ledger rows.
|
||||
pub fn bootstrap_median_ci(sample: &[f64], iters: usize, confidence: f64, seed: u64) -> (f64, f64, f64) {
|
||||
pub fn bootstrap_median_ci(
|
||||
sample: &[f64],
|
||||
iters: usize,
|
||||
confidence: f64,
|
||||
seed: u64,
|
||||
) -> (f64, f64, f64) {
|
||||
assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence));
|
||||
let mut rng = XorShift64::new(seed);
|
||||
let mut medians = Vec::with_capacity(iters);
|
||||
@@ -106,7 +114,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn median_ci_brackets_median() {
|
||||
let s: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
|
||||
let s: Vec<f64> = (0..40)
|
||||
.map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64)
|
||||
.collect();
|
||||
let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9);
|
||||
assert!(lo <= m && m <= hi);
|
||||
}
|
||||
@@ -120,15 +130,22 @@ mod tests {
|
||||
#[test]
|
||||
fn aa_ci_straddles_one() {
|
||||
// Same distribution both sides: the CI must contain 1.0.
|
||||
let sample: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
|
||||
let sample: Vec<f64> = (0..40)
|
||||
.map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64)
|
||||
.collect();
|
||||
let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42);
|
||||
assert!(ci.lo <= 1.0 && ci.hi >= 1.0, "A/A CI must straddle 1.0, got {ci:?}");
|
||||
assert!(
|
||||
ci.lo <= 1.0 && ci.hi >= 1.0,
|
||||
"A/A CI must straddle 1.0, got {ci:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_injected_regression() {
|
||||
// Candidate 5% slower with mild noise: speedup CI entirely < 1.
|
||||
let base: Vec<f64> = (0..40).map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64).collect();
|
||||
let base: Vec<f64> = (0..40)
|
||||
.map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64)
|
||||
.collect();
|
||||
let cand: Vec<f64> = base.iter().map(|t| t * 1.05).collect();
|
||||
let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42);
|
||||
assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}");
|
||||
|
||||
Reference in New Issue
Block a user