benbrown/peltier.io
2
1
Fork 0
mirror of https://github.com/bdbrown4/peltier.io.git synced 2026-11-07 10:58:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

bench: GitHub Actions CI — trust-layer checks, no-net gates, runner calibration

Browse Source 
Closes the CI-reachable slice of the Phase 1 infra gaps:
- trust-layer job: cargo test + clippy -D warnings + fmt
- target-gates-no-net: fetches pinned targets (new targets/fetch.sh),
  regenerates + verifies corpora, runs 'just gates tokei'
  mechanically, and executes the bench workload inside a
  --network=none container (SPEC 3.1/10 no-net requirement)
- runner-calibration (informational): A/A + injection harness measures
  the shared runner's noise floor per run, evidence uploaded as an
  artifact — environments are judged empirically, not assumed
- perf-probe (informational): reports which perf event sources work

docs/infra.md maps each remaining hardware requirement (pinned cores,
governor/SMT control, separate-uid trust layer) to the bench-metal
self-hosted runner setup a human can complete in ~15 min.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
Claude
2026-07-07 00:13:21 +00:00
parent 19e83d878a
commit 52089c0451
12 changed files with 375 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bench-runner/src/fingerprint.rs
+1 -3
View File
@@ -38,9 +38,7 @@ impl EnvFingerprint {
Self {
kernel: read_trimmed("/proc/sys/kernel/osrelease"),
cpu_model,
governor: read_trimmed(
"/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
),
governor: read_trimmed("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"),
smt: read_trimmed("/sys/devices/system/cpu/smt/active"),
turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"),
aslr: format!(
crates/bench-runner/src/lib.rs
+10 -2
View File
@@ -34,10 +34,18 @@ mod tests {
#[test]
fn accept_requires_ci_lower_bound_not_point_estimate() {
// 10% median speedup but the CI dips below threshold: rejected.
let noisy = RatioCi { median: 1.10, lo: 1.005, hi: 1.21 };
let noisy = RatioCi {
median: 1.10,
lo: 1.005,
hi: 1.21,
};
assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench);
// 4% speedup with a tight CI: accepted.
let tight = RatioCi { median: 1.04, lo: 1.025, hi: 1.055 };
let tight = RatioCi {
median: 1.04,
lo: 1.025,
hi: 1.055,
};
assert_eq!(decide(tight, 0.02), Verdict::Accepted);
}
}
crates/bench-runner/src/main.rs
+36 -11
View File
@@ -5,7 +5,10 @@ use ledger::Verdict;
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")]
#[command(
name = "bench-runner",
about = "Interleaved A/B benchmark harness (hotpath trust layer)"
)]
struct Cli {
/// Path to accept.toml; built-in defaults if omitted.
#[arg(long)]
@@ -68,23 +71,32 @@ fn calibrate(
let seed = cfg.bootstrap_seed.wrapping_add(i as u64);
let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
let aa_ci = stats::bootstrap_ratio_ci(
&aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
&aa.baseline_s,
&aa.candidate_s,
cfg.bootstrap_iters,
cfg.confidence,
seed,
);
if decide(aa_ci, cfg.threshold) == Verdict::Accepted {
false_positives += 1;
}
aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
aa_sessions
.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
let inj =
exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
let inj = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
let inj_ci = stats::bootstrap_ratio_ci(
&inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
&inj.baseline_s,
&inj.candidate_s,
cfg.bootstrap_iters,
cfg.confidence,
seed,
);
// Detected = the harness resolves a regression: CI entirely < 1.
if inj_ci.hi < 1.0 {
detections += 1;
}
inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
inj_sessions
.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
eprintln!(
"session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}",
i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi
@@ -127,9 +139,17 @@ fn main() -> Result<()> {
};
let (baseline, candidate, aa_mode) = match &cli.command {
Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
Cmd::Compare {
baseline,
candidate,
} => (baseline.clone(), candidate.clone(), false),
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
Cmd::Calibrate { cmd, sessions, slowdown, out } => {
Cmd::Calibrate {
cmd,
sessions,
slowdown,
out,
} => {
return calibrate(&cfg, cmd, *sessions, *slowdown, out);
}
};
@@ -141,8 +161,13 @@ fn main() -> Result<()> {
cfg.runs_per_side, cfg.warmup_runs
);
let samples =
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
let samples = exec::run_interleaved(
&baseline,
&candidate,
cfg.runs_per_side,
cfg.warmup_runs,
0.0,
)?;
let ci = stats::bootstrap_ratio_ci(
&samples.baseline_s,
&samples.candidate_s,
crates/bench-runner/src/stats.rs
+23 -6
View File
@@ -54,7 +54,10 @@ pub fn bootstrap_ratio_ci(
confidence: f64,
seed: u64,
) -> RatioCi {
assert!(baseline.len() >= 2 && candidate.len() >= 2, "need >=2 runs per side");
assert!(
baseline.len() >= 2 && candidate.len() >= 2,
"need >=2 runs per side"
);
assert!((0.0..1.0).contains(&confidence));
let mut rng = XorShift64::new(seed);
let mut ratios = Vec::with_capacity(iters);
@@ -82,7 +85,12 @@ pub fn bootstrap_ratio_ci(
/// Percentile-method bootstrap CI of a single sample's median, for
/// per-side evidence in ledger rows.
pub fn bootstrap_median_ci(sample: &[f64], iters: usize, confidence: f64, seed: u64) -> (f64, f64, f64) {
pub fn bootstrap_median_ci(
sample: &[f64],
iters: usize,
confidence: f64,
seed: u64,
) -> (f64, f64, f64) {
assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence));
let mut rng = XorShift64::new(seed);
let mut medians = Vec::with_capacity(iters);
@@ -106,7 +114,9 @@ mod tests {
#[test]
fn median_ci_brackets_median() {
let s: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
let s: Vec<f64> = (0..40)
.map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64)
.collect();
let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9);
assert!(lo <= m && m <= hi);
}
@@ -120,15 +130,22 @@ mod tests {
#[test]
fn aa_ci_straddles_one() {
// Same distribution both sides: the CI must contain 1.0.
let sample: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
let sample: Vec<f64> = (0..40)
.map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64)
.collect();
let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42);
assert!(ci.lo <= 1.0 && ci.hi >= 1.0, "A/A CI must straddle 1.0, got {ci:?}");
assert!(
ci.lo <= 1.0 && ci.hi >= 1.0,
"A/A CI must straddle 1.0, got {ci:?}"
);
}
#[test]
fn detects_injected_regression() {
// Candidate 5% slower with mild noise: speedup CI entirely < 1.
let base: Vec<f64> = (0..40).map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64).collect();
let base: Vec<f64> = (0..40)
.map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64)
.collect();
let cand: Vec<f64> = base.iter().map(|t| t * 1.05).collect();
let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42);
assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}");
crates/diff-test/src/lib.rs
+12 -7
View File
@@ -9,8 +9,8 @@
pub mod orchestrate;
pub mod pin;
pub mod target;
pub mod policy;
pub mod target;
use serde::{Deserialize, Serialize};
@@ -35,11 +35,12 @@ pub enum GateLayer {
/// All layers must pass; any failure is an auto-reject (SPEC §8).
pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool {
results.iter().all(|(_, o)| {
matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. })
}) && results
results
.iter()
.any(|(_, o)| matches!(o, GateOutcome::Passed))
.all(|(_, o)| matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. }))
&& results
.iter()
.any(|(_, o)| matches!(o, GateOutcome::Passed))
}
#[cfg(test)]
@@ -52,7 +53,9 @@ mod tests {
(GateLayer::UpstreamTests, GateOutcome::Passed),
(
GateLayer::GoldenReplay,
GateOutcome::Failed { detail: "stdout diverged".into() },
GateOutcome::Failed {
detail: "stdout diverged".into(),
},
),
];
assert!(!all_passed(&results));
@@ -62,7 +65,9 @@ mod tests {
fn all_skipped_is_not_a_pass() {
let results = vec![(
GateLayer::Sanitizers,
GateOutcome::Skipped { reason: "n/a".into() },
GateOutcome::Skipped {
reason: "n/a".into(),
},
)];
assert!(!all_passed(&results));
}
crates/diff-test/src/orchestrate.rs
+8 -2
View File
@@ -43,7 +43,10 @@ pub fn run_core_gates(
GateOutcome::Passed
} else {
GateOutcome::Failed {
detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(),
detail: String::from_utf8_lossy(&t.stderr)
.chars()
.take(2000)
.collect(),
}
},
));
@@ -57,7 +60,10 @@ pub fn run_core_gates(
GateOutcome::Passed
} else {
GateOutcome::Failed {
detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()),
detail: format!(
"exit={:?} expected={expected} actual={actual}",
g.status.code()
),
}
},
));
crates/diff-test/src/policy.rs
+4 -1
View File
@@ -67,7 +67,10 @@ mod tests {
.unwrap();
assert_eq!(
EquivalencePolicy::load(&dir).unwrap(),
EquivalencePolicy::FpTolerance { abs: 1e-9, rel: 1e-6 }
EquivalencePolicy::FpTolerance {
abs: 1e-9,
rel: 1e-6
}
);
}
}
crates/harnessd/src/main.rs
+49 -15
View File
@@ -41,7 +41,10 @@ fn field<'a>(req: &'a Value, key: &str) -> Result<&'a str> {
/// A target name must be a plain directory name — no separators, no dots.
fn check_target(name: &str) -> Result<()> {
ensure!(
!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
!name.is_empty()
&& name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
"invalid target name"
);
Ok(())
@@ -88,7 +91,11 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
let dir = root.join("playbook");
let entry = std::fs::read_dir(&dir)?
.filter_map(|e| e.ok())
.find(|e| e.file_name().to_string_lossy().starts_with(&format!("{class:02}-")))
.find(|e| {
e.file_name()
.to_string_lossy()
.starts_with(&format!("{class:02}-"))
})
.ok_or_else(|| anyhow!("no playbook file for class {class}"))?;
Ok(json!({"markdown": std::fs::read_to_string(entry.path())?}))
}
@@ -109,10 +116,16 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
// relative path (the git -C below roots them in the target
// workspace; nothing outside it is reachable).
for l in diff.lines() {
if let Some(p) = l.strip_prefix("--- a/").or_else(|| l.strip_prefix("+++ b/")) {
if let Some(p) = l
.strip_prefix("--- a/")
.or_else(|| l.strip_prefix("+++ b/"))
{
check_rel_path(p.trim())?;
}
ensure!(!l.starts_with("--- /") && !l.starts_with("+++ /"), "absolute diff paths forbidden");
ensure!(
!l.starts_with("--- /") && !l.starts_with("+++ /"),
"absolute diff paths forbidden"
);
}
let ws = root.join(format!("targets/{t}/workspace"));
let mut check = Command::new("git")
@@ -132,13 +145,18 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
std::fs::create_dir_all(&pending)?;
std::fs::write(
pending.join(format!("{patch_id}.json")),
serde_json::to_string_pretty(&json!({"target": t, "hypothesis": hypothesis, "diff": diff}))?,
serde_json::to_string_pretty(
&json!({"target": t, "hypothesis": hypothesis, "diff": diff}),
)?,
)?;
Ok(json!({"patch_id": patch_id}))
}
"run_verdict" => {
let patch_id = field(&req, "patch_id")?;
ensure!(patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12, "bad patch id");
ensure!(
patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12,
"bad patch id"
);
let run_id = field(&req, "run_id")?;
let class = field(&req, "playbook_class")?;
let hotspot = field(&req, "hotspot")?;
@@ -151,24 +169,40 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
// isolated dir, then hand off to the verdict pipeline.
let cand_dir = root.join(format!("targets/{t}/candidate-{patch_id}"));
let build = Command::new("sh")
.arg("-c").arg(&spec.build.baseline)
.arg("-c")
.arg(&spec.build.baseline)
.current_dir(root)
.env("CARGO_TARGET_DIR", &cand_dir)
.status()?;
ensure!(build.success(), "candidate build failed");
let bin = Path::new(&spec.build.binary).file_name().unwrap().to_str().unwrap();
let bin = Path::new(&spec.build.binary)
.file_name()
.unwrap()
.to_str()
.unwrap();
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
let out = Command::new("cargo")
.args([
"run", "-q", "-p", "verdict", "--", t,
"run",
"-q",
"-p",
"verdict",
"--",
t,
"--rebuild-baseline",
"--candidate-bin", &format!("targets/{t}/candidate-{patch_id}/release/{bin}"),
"--run-id", run_id,
"--playbook-class", class,
"--hypothesis", pending["hypothesis"].as_str().unwrap(),
"--hotspot", hotspot,
"--patch-file", diff_path.to_str().unwrap(),
"--candidate-bin",
&format!("targets/{t}/candidate-{patch_id}/release/{bin}"),
"--run-id",
run_id,
"--playbook-class",
class,
"--hypothesis",
pending["hypothesis"].as_str().unwrap(),
"--hotspot",
hotspot,
"--patch-file",
diff_path.to_str().unwrap(),
])
.current_dir(root)
.output()?;
crates/verdict/src/main.rs
+84 -46
View File
@@ -8,12 +8,17 @@
use anyhow::Result;
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
use clap::Parser;
use diff_test::{all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome};
use diff_test::{
all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome,
};
use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict};
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "verdict", about = "gates + bench + ledger row for one optimization attempt")]
#[command(
name = "verdict",
about = "gates + bench + ledger row for one optimization attempt"
)]
struct Cli {
/// Target name under targets/.
target: String,
@@ -78,7 +83,10 @@ fn rebuild_pristine_baseline(
.arg("-c")
.arg(&spec.build.baseline)
.current_dir(root)
.env("CARGO_TARGET_DIR", root.join(format!("targets/{target}/baseline")))
.env(
"CARGO_TARGET_DIR",
root.join(format!("targets/{target}/baseline")),
)
.status();
if dirty {
anyhow::ensure!(
@@ -95,7 +103,10 @@ fn rebuild_pristine_baseline(
.and_then(|n| n.to_str())
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
let baseline = format!("targets/{target}/baseline/release/{bin_name}");
anyhow::ensure!(root.join(&baseline).exists(), "baseline binary missing: {baseline}");
anyhow::ensure!(
root.join(&baseline).exists(),
"baseline binary missing: {baseline}"
);
println!("baseline rebuilt from pristine checkout: {baseline}");
Ok(baseline)
}
@@ -147,52 +158,75 @@ fn main() -> Result<()> {
let pin = &cfg.pin_prefix;
let wrap = |bin: &str| {
let cmd = spec.bench.command.replace("{binary}", bin);
if pin.is_empty() { format!("sh -c \"{cmd}\"") } else { format!("{pin} sh -c \"{cmd}\"") }
if pin.is_empty() {
format!("sh -c \"{cmd}\"")
} else {
format!("{pin} sh -c \"{cmd}\"")
}
};
let (verdict, bench) = if !all_passed(&gates) {
println!("verdict: rejected-gate (bench skipped)");
(Verdict::RejectedGate, None)
} else {
println!(
"bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'",
cfg.runs_per_side, cfg.warmup_runs, pin
);
let samples = exec::run_interleaved(
&wrap(&baseline_bin), &wrap(&cli.candidate_bin),
cfg.runs_per_side, cfg.warmup_runs, 0.0,
)?;
let ratio = stats::bootstrap_ratio_ci(
&samples.baseline_s, &samples.candidate_s,
cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
);
let (bm, blo, bhi) = stats::bootstrap_median_ci(
&samples.baseline_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
);
let (cm, clo, chi) = stats::bootstrap_median_ci(
&samples.candidate_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
);
println!(
let (verdict, bench) =
if !all_passed(&gates) {
println!("verdict: rejected-gate (bench skipped)");
(Verdict::RejectedGate, None)
} else {
println!(
"bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'",
cfg.runs_per_side, cfg.warmup_runs, pin
);
let samples = exec::run_interleaved(
&wrap(&baseline_bin),
&wrap(&cli.candidate_bin),
cfg.runs_per_side,
cfg.warmup_runs,
0.0,
)?;
let ratio = stats::bootstrap_ratio_ci(
&samples.baseline_s,
&samples.candidate_s,
cfg.bootstrap_iters,
cfg.confidence,
cfg.bootstrap_seed,
);
let (bm, blo, bhi) = stats::bootstrap_median_ci(
&samples.baseline_s,
cfg.bootstrap_iters,
cfg.confidence,
cfg.bootstrap_seed,
);
let (cm, clo, chi) = stats::bootstrap_median_ci(
&samples.candidate_s,
cfg.bootstrap_iters,
cfg.confidence,
cfg.bootstrap_seed,
);
println!(
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload
);
let mut v = decide(ratio, cfg.threshold);
if cli.needs_human_review && v == Verdict::Accepted {
v = Verdict::NeedsHumanReview;
}
let fp = EnvFingerprint::collect(pin, "system-default");
(v, Some(BenchEvidence {
baseline_median: bm, baseline_ci: (blo, bhi),
candidate_median: cm, candidate_ci: (clo, chi),
speedup_median: ratio.median, speedup_ci: (ratio.lo, ratio.hi),
env_fingerprint: serde_json::json!({
"fingerprint": fp,
"workload": spec.bench.workload,
"target_commit": spec.source.commit,
"gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
}),
}))
};
let mut v = decide(ratio, cfg.threshold);
if cli.needs_human_review && v == Verdict::Accepted {
v = Verdict::NeedsHumanReview;
}
let fp = EnvFingerprint::collect(pin, "system-default");
(
v,
Some(BenchEvidence {
baseline_median: bm,
baseline_ci: (blo, bhi),
candidate_median: cm,
candidate_ci: (clo, chi),
speedup_median: ratio.median,
speedup_ci: (ratio.lo, ratio.hi),
env_fingerprint: serde_json::json!({
"fingerprint": fp,
"workload": spec.bench.workload,
"target_commit": spec.source.commit,
"gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
}),
}),
)
};
let patch = match &cli.patch_file {
Some(p) => std::fs::read_to_string(p)?,
@@ -215,6 +249,10 @@ fn main() -> Result<()> {
wall_time_s: started.elapsed().as_secs_f64(),
};
Ledger::open(&root.join(&cli.db))?.record(&attempt)?;
println!("verdict: {} (ledger row {})", attempt.verdict.as_str(), attempt.run_id);
println!(
"verdict: {} (ledger row {})",
attempt.verdict.as_str(),
attempt.run_id
);
Ok(())
}