diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c4ca94f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,87 @@ +name: ci +on: + pull_request: + push: + branches: [main] + +jobs: + trust-layer: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test --workspace + - run: cargo clippy --workspace --all-targets -- -D warnings + - run: cargo fmt --all --check + + # Phase 1: target gates run mechanically, and the bench workload + # executes inside a NO-NETWORK container (SPEC §3.1/§10). The runner + # is a shared VM — fine for gates; bench numbers for verdicts come + # from calibrated hardware, not this job. + target-gates-no-net: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Fetch pinned targets + run: | + sh targets/fetch.sh comrak + sh targets/fetch.sh tokei + - name: Generate + verify pinned corpora + run: | + sh corpora/comrak/gen-corpus.sh + sh corpora/tokei/gen-corpus.sh + - name: Build tokei baseline + run: cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml + - name: Gates (corpus pin -> upstream tests -> golden replay) + run: cargo run -p diff-test -- tokei + - name: Bench workload inside no-network container + run: | + docker run --rm --network=none \ + -v "$PWD:/w" -w /w \ + debian:bookworm-slim \ + sh -c "targets/tokei/workspace/target/release/tokei --sort code corpora/tokei/input | sha256sum" + + # Informational: measure THIS runner's noise floor with the A/A + + # injection harness and archive the evidence. A shared VM failing the + # bars is expected — the artifact is the point (trust layer judges + # environments empirically; verdict benches belong on bench-metal). + runner-calibration: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Calibrate (reduced sessions, synthetic CPU-bound workload) + run: | + mkdir -p calib + cargo run -p bench-runner -- calibrate \ + --cmd "head -c 3000000 /dev/urandom | sha256sum" \ + --sessions 8 --out calib/gh-runner-calibration.json \ + | tee calib/summary.txt + - uses: actions/upload-artifact@v4 + if: always() + with: + name: runner-calibration + path: calib/ + + # Informational: does perf work here, and with which event sources? + perf-probe: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Probe perf availability + run: | + sudo apt-get -qq update && sudo apt-get -qq install -y linux-tools-common linux-tools-generic || true + PERF=$(ls /usr/lib/linux-tools/*/perf 2>/dev/null | head -1 || command -v perf || true) + echo "perf binary: ${PERF:-none}" + [ -n "$PERF" ] || exit 0 + sudo sysctl -w kernel.perf_event_paranoid=1 || true + echo "--- software clock sampling:" + $PERF record -e cpu-clock -o /tmp/p.data -- sh -c 'i=0; while [ $i -lt 2000000 ]; do i=$((i+1)); done' && $PERF report -i /tmp/p.data --stdio | head -5 || echo "cpu-clock sampling unavailable" + echo "--- hardware cycle counter:" + $PERF stat -e cycles true 2>&1 | tail -3 || true diff --git a/crates/bench-runner/src/fingerprint.rs b/crates/bench-runner/src/fingerprint.rs index 683593a..f74e7bf 100644 --- a/crates/bench-runner/src/fingerprint.rs +++ b/crates/bench-runner/src/fingerprint.rs @@ -38,9 +38,7 @@ impl EnvFingerprint { Self { kernel: read_trimmed("/proc/sys/kernel/osrelease"), cpu_model, - governor: read_trimmed( - "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", - ), + governor: read_trimmed("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"), smt: read_trimmed("/sys/devices/system/cpu/smt/active"), turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"), aslr: format!( diff --git a/crates/bench-runner/src/lib.rs b/crates/bench-runner/src/lib.rs index c2b5340..bb789a4 100644 --- a/crates/bench-runner/src/lib.rs +++ b/crates/bench-runner/src/lib.rs @@ -34,10 +34,18 @@ mod tests { #[test] fn accept_requires_ci_lower_bound_not_point_estimate() { // 10% median speedup but the CI dips below threshold: rejected. - let noisy = RatioCi { median: 1.10, lo: 1.005, hi: 1.21 }; + let noisy = RatioCi { + median: 1.10, + lo: 1.005, + hi: 1.21, + }; assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench); // 4% speedup with a tight CI: accepted. - let tight = RatioCi { median: 1.04, lo: 1.025, hi: 1.055 }; + let tight = RatioCi { + median: 1.04, + lo: 1.025, + hi: 1.055, + }; assert_eq!(decide(tight, 0.02), Verdict::Accepted); } } diff --git a/crates/bench-runner/src/main.rs b/crates/bench-runner/src/main.rs index 1e79e89..e79f95d 100644 --- a/crates/bench-runner/src/main.rs +++ b/crates/bench-runner/src/main.rs @@ -5,7 +5,10 @@ use ledger::Verdict; use std::path::PathBuf; #[derive(Parser)] -#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")] +#[command( + name = "bench-runner", + about = "Interleaved A/B benchmark harness (hotpath trust layer)" +)] struct Cli { /// Path to accept.toml; built-in defaults if omitted. #[arg(long)] @@ -68,23 +71,32 @@ fn calibrate( let seed = cfg.bootstrap_seed.wrapping_add(i as u64); let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?; let aa_ci = stats::bootstrap_ratio_ci( - &aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed, + &aa.baseline_s, + &aa.candidate_s, + cfg.bootstrap_iters, + cfg.confidence, + seed, ); if decide(aa_ci, cfg.threshold) == Verdict::Accepted { false_positives += 1; } - aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi})); + aa_sessions + .push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi})); - let inj = - exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?; + let inj = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?; let inj_ci = stats::bootstrap_ratio_ci( - &inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed, + &inj.baseline_s, + &inj.candidate_s, + cfg.bootstrap_iters, + cfg.confidence, + seed, ); // Detected = the harness resolves a regression: CI entirely < 1. if inj_ci.hi < 1.0 { detections += 1; } - inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi})); + inj_sessions + .push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi})); eprintln!( "session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}", i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi @@ -127,9 +139,17 @@ fn main() -> Result<()> { }; let (baseline, candidate, aa_mode) = match &cli.command { - Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false), + Cmd::Compare { + baseline, + candidate, + } => (baseline.clone(), candidate.clone(), false), Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true), - Cmd::Calibrate { cmd, sessions, slowdown, out } => { + Cmd::Calibrate { + cmd, + sessions, + slowdown, + out, + } => { return calibrate(&cfg, cmd, *sessions, *slowdown, out); } }; @@ -141,8 +161,13 @@ fn main() -> Result<()> { cfg.runs_per_side, cfg.warmup_runs ); - let samples = - exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?; + let samples = exec::run_interleaved( + &baseline, + &candidate, + cfg.runs_per_side, + cfg.warmup_runs, + 0.0, + )?; let ci = stats::bootstrap_ratio_ci( &samples.baseline_s, &samples.candidate_s, diff --git a/crates/bench-runner/src/stats.rs b/crates/bench-runner/src/stats.rs index 53576e0..79d7304 100644 --- a/crates/bench-runner/src/stats.rs +++ b/crates/bench-runner/src/stats.rs @@ -54,7 +54,10 @@ pub fn bootstrap_ratio_ci( confidence: f64, seed: u64, ) -> RatioCi { - assert!(baseline.len() >= 2 && candidate.len() >= 2, "need >=2 runs per side"); + assert!( + baseline.len() >= 2 && candidate.len() >= 2, + "need >=2 runs per side" + ); assert!((0.0..1.0).contains(&confidence)); let mut rng = XorShift64::new(seed); let mut ratios = Vec::with_capacity(iters); @@ -82,7 +85,12 @@ pub fn bootstrap_ratio_ci( /// Percentile-method bootstrap CI of a single sample's median, for /// per-side evidence in ledger rows. -pub fn bootstrap_median_ci(sample: &[f64], iters: usize, confidence: f64, seed: u64) -> (f64, f64, f64) { +pub fn bootstrap_median_ci( + sample: &[f64], + iters: usize, + confidence: f64, + seed: u64, +) -> (f64, f64, f64) { assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence)); let mut rng = XorShift64::new(seed); let mut medians = Vec::with_capacity(iters); @@ -106,7 +114,9 @@ mod tests { #[test] fn median_ci_brackets_median() { - let s: Vec = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect(); + let s: Vec = (0..40) + .map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64) + .collect(); let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9); assert!(lo <= m && m <= hi); } @@ -120,15 +130,22 @@ mod tests { #[test] fn aa_ci_straddles_one() { // Same distribution both sides: the CI must contain 1.0. - let sample: Vec = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect(); + let sample: Vec = (0..40) + .map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64) + .collect(); let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42); - assert!(ci.lo <= 1.0 && ci.hi >= 1.0, "A/A CI must straddle 1.0, got {ci:?}"); + assert!( + ci.lo <= 1.0 && ci.hi >= 1.0, + "A/A CI must straddle 1.0, got {ci:?}" + ); } #[test] fn detects_injected_regression() { // Candidate 5% slower with mild noise: speedup CI entirely < 1. - let base: Vec = (0..40).map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64).collect(); + let base: Vec = (0..40) + .map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64) + .collect(); let cand: Vec = base.iter().map(|t| t * 1.05).collect(); let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42); assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}"); diff --git a/crates/diff-test/src/lib.rs b/crates/diff-test/src/lib.rs index 131e8db..a769a4e 100644 --- a/crates/diff-test/src/lib.rs +++ b/crates/diff-test/src/lib.rs @@ -9,8 +9,8 @@ pub mod orchestrate; pub mod pin; -pub mod target; pub mod policy; +pub mod target; use serde::{Deserialize, Serialize}; @@ -35,11 +35,12 @@ pub enum GateLayer { /// All layers must pass; any failure is an auto-reject (SPEC §8). pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool { - results.iter().all(|(_, o)| { - matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. }) - }) && results + results .iter() - .any(|(_, o)| matches!(o, GateOutcome::Passed)) + .all(|(_, o)| matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. })) + && results + .iter() + .any(|(_, o)| matches!(o, GateOutcome::Passed)) } #[cfg(test)] @@ -52,7 +53,9 @@ mod tests { (GateLayer::UpstreamTests, GateOutcome::Passed), ( GateLayer::GoldenReplay, - GateOutcome::Failed { detail: "stdout diverged".into() }, + GateOutcome::Failed { + detail: "stdout diverged".into(), + }, ), ]; assert!(!all_passed(&results)); @@ -62,7 +65,9 @@ mod tests { fn all_skipped_is_not_a_pass() { let results = vec![( GateLayer::Sanitizers, - GateOutcome::Skipped { reason: "n/a".into() }, + GateOutcome::Skipped { + reason: "n/a".into(), + }, )]; assert!(!all_passed(&results)); } diff --git a/crates/diff-test/src/orchestrate.rs b/crates/diff-test/src/orchestrate.rs index b140b77..5825cf9 100644 --- a/crates/diff-test/src/orchestrate.rs +++ b/crates/diff-test/src/orchestrate.rs @@ -43,7 +43,10 @@ pub fn run_core_gates( GateOutcome::Passed } else { GateOutcome::Failed { - detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(), + detail: String::from_utf8_lossy(&t.stderr) + .chars() + .take(2000) + .collect(), } }, )); @@ -57,7 +60,10 @@ pub fn run_core_gates( GateOutcome::Passed } else { GateOutcome::Failed { - detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()), + detail: format!( + "exit={:?} expected={expected} actual={actual}", + g.status.code() + ), } }, )); diff --git a/crates/diff-test/src/policy.rs b/crates/diff-test/src/policy.rs index ff9a5b7..7a2988f 100644 --- a/crates/diff-test/src/policy.rs +++ b/crates/diff-test/src/policy.rs @@ -67,7 +67,10 @@ mod tests { .unwrap(); assert_eq!( EquivalencePolicy::load(&dir).unwrap(), - EquivalencePolicy::FpTolerance { abs: 1e-9, rel: 1e-6 } + EquivalencePolicy::FpTolerance { + abs: 1e-9, + rel: 1e-6 + } ); } } diff --git a/crates/harnessd/src/main.rs b/crates/harnessd/src/main.rs index 514bb72..be74189 100644 --- a/crates/harnessd/src/main.rs +++ b/crates/harnessd/src/main.rs @@ -41,7 +41,10 @@ fn field<'a>(req: &'a Value, key: &str) -> Result<&'a str> { /// A target name must be a plain directory name — no separators, no dots. fn check_target(name: &str) -> Result<()> { ensure!( - !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'), + !name.is_empty() + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'), "invalid target name" ); Ok(()) @@ -88,7 +91,11 @@ fn handle(root: &Path, line: &str) -> Result { let dir = root.join("playbook"); let entry = std::fs::read_dir(&dir)? .filter_map(|e| e.ok()) - .find(|e| e.file_name().to_string_lossy().starts_with(&format!("{class:02}-"))) + .find(|e| { + e.file_name() + .to_string_lossy() + .starts_with(&format!("{class:02}-")) + }) .ok_or_else(|| anyhow!("no playbook file for class {class}"))?; Ok(json!({"markdown": std::fs::read_to_string(entry.path())?})) } @@ -109,10 +116,16 @@ fn handle(root: &Path, line: &str) -> Result { // relative path (the git -C below roots them in the target // workspace; nothing outside it is reachable). for l in diff.lines() { - if let Some(p) = l.strip_prefix("--- a/").or_else(|| l.strip_prefix("+++ b/")) { + if let Some(p) = l + .strip_prefix("--- a/") + .or_else(|| l.strip_prefix("+++ b/")) + { check_rel_path(p.trim())?; } - ensure!(!l.starts_with("--- /") && !l.starts_with("+++ /"), "absolute diff paths forbidden"); + ensure!( + !l.starts_with("--- /") && !l.starts_with("+++ /"), + "absolute diff paths forbidden" + ); } let ws = root.join(format!("targets/{t}/workspace")); let mut check = Command::new("git") @@ -132,13 +145,18 @@ fn handle(root: &Path, line: &str) -> Result { std::fs::create_dir_all(&pending)?; std::fs::write( pending.join(format!("{patch_id}.json")), - serde_json::to_string_pretty(&json!({"target": t, "hypothesis": hypothesis, "diff": diff}))?, + serde_json::to_string_pretty( + &json!({"target": t, "hypothesis": hypothesis, "diff": diff}), + )?, )?; Ok(json!({"patch_id": patch_id})) } "run_verdict" => { let patch_id = field(&req, "patch_id")?; - ensure!(patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12, "bad patch id"); + ensure!( + patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12, + "bad patch id" + ); let run_id = field(&req, "run_id")?; let class = field(&req, "playbook_class")?; let hotspot = field(&req, "hotspot")?; @@ -151,24 +169,40 @@ fn handle(root: &Path, line: &str) -> Result { // isolated dir, then hand off to the verdict pipeline. let cand_dir = root.join(format!("targets/{t}/candidate-{patch_id}")); let build = Command::new("sh") - .arg("-c").arg(&spec.build.baseline) + .arg("-c") + .arg(&spec.build.baseline) .current_dir(root) .env("CARGO_TARGET_DIR", &cand_dir) .status()?; ensure!(build.success(), "candidate build failed"); - let bin = Path::new(&spec.build.binary).file_name().unwrap().to_str().unwrap(); + let bin = Path::new(&spec.build.binary) + .file_name() + .unwrap() + .to_str() + .unwrap(); let diff_path = root.join(format!("results/pending/{patch_id}.json.diff")); std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?; let out = Command::new("cargo") .args([ - "run", "-q", "-p", "verdict", "--", t, + "run", + "-q", + "-p", + "verdict", + "--", + t, "--rebuild-baseline", - "--candidate-bin", &format!("targets/{t}/candidate-{patch_id}/release/{bin}"), - "--run-id", run_id, - "--playbook-class", class, - "--hypothesis", pending["hypothesis"].as_str().unwrap(), - "--hotspot", hotspot, - "--patch-file", diff_path.to_str().unwrap(), + "--candidate-bin", + &format!("targets/{t}/candidate-{patch_id}/release/{bin}"), + "--run-id", + run_id, + "--playbook-class", + class, + "--hypothesis", + pending["hypothesis"].as_str().unwrap(), + "--hotspot", + hotspot, + "--patch-file", + diff_path.to_str().unwrap(), ]) .current_dir(root) .output()?; diff --git a/crates/verdict/src/main.rs b/crates/verdict/src/main.rs index 999ad3c..5ed1dfe 100644 --- a/crates/verdict/src/main.rs +++ b/crates/verdict/src/main.rs @@ -8,12 +8,17 @@ use anyhow::Result; use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats}; use clap::Parser; -use diff_test::{all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome}; +use diff_test::{ + all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome, +}; use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict}; use std::path::PathBuf; #[derive(Parser)] -#[command(name = "verdict", about = "gates + bench + ledger row for one optimization attempt")] +#[command( + name = "verdict", + about = "gates + bench + ledger row for one optimization attempt" +)] struct Cli { /// Target name under targets/. target: String, @@ -78,7 +83,10 @@ fn rebuild_pristine_baseline( .arg("-c") .arg(&spec.build.baseline) .current_dir(root) - .env("CARGO_TARGET_DIR", root.join(format!("targets/{target}/baseline"))) + .env( + "CARGO_TARGET_DIR", + root.join(format!("targets/{target}/baseline")), + ) .status(); if dirty { anyhow::ensure!( @@ -95,7 +103,10 @@ fn rebuild_pristine_baseline( .and_then(|n| n.to_str()) .ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?; let baseline = format!("targets/{target}/baseline/release/{bin_name}"); - anyhow::ensure!(root.join(&baseline).exists(), "baseline binary missing: {baseline}"); + anyhow::ensure!( + root.join(&baseline).exists(), + "baseline binary missing: {baseline}" + ); println!("baseline rebuilt from pristine checkout: {baseline}"); Ok(baseline) } @@ -147,52 +158,75 @@ fn main() -> Result<()> { let pin = &cfg.pin_prefix; let wrap = |bin: &str| { let cmd = spec.bench.command.replace("{binary}", bin); - if pin.is_empty() { format!("sh -c \"{cmd}\"") } else { format!("{pin} sh -c \"{cmd}\"") } + if pin.is_empty() { + format!("sh -c \"{cmd}\"") + } else { + format!("{pin} sh -c \"{cmd}\"") + } }; - let (verdict, bench) = if !all_passed(&gates) { - println!("verdict: rejected-gate (bench skipped)"); - (Verdict::RejectedGate, None) - } else { - println!( - "bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'", - cfg.runs_per_side, cfg.warmup_runs, pin - ); - let samples = exec::run_interleaved( - &wrap(&baseline_bin), &wrap(&cli.candidate_bin), - cfg.runs_per_side, cfg.warmup_runs, 0.0, - )?; - let ratio = stats::bootstrap_ratio_ci( - &samples.baseline_s, &samples.candidate_s, - cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed, - ); - let (bm, blo, bhi) = stats::bootstrap_median_ci( - &samples.baseline_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed, - ); - let (cm, clo, chi) = stats::bootstrap_median_ci( - &samples.candidate_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed, - ); - println!( + let (verdict, bench) = + if !all_passed(&gates) { + println!("verdict: rejected-gate (bench skipped)"); + (Verdict::RejectedGate, None) + } else { + println!( + "bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'", + cfg.runs_per_side, cfg.warmup_runs, pin + ); + let samples = exec::run_interleaved( + &wrap(&baseline_bin), + &wrap(&cli.candidate_bin), + cfg.runs_per_side, + cfg.warmup_runs, + 0.0, + )?; + let ratio = stats::bootstrap_ratio_ci( + &samples.baseline_s, + &samples.candidate_s, + cfg.bootstrap_iters, + cfg.confidence, + cfg.bootstrap_seed, + ); + let (bm, blo, bhi) = stats::bootstrap_median_ci( + &samples.baseline_s, + cfg.bootstrap_iters, + cfg.confidence, + cfg.bootstrap_seed, + ); + let (cm, clo, chi) = stats::bootstrap_median_ci( + &samples.candidate_s, + cfg.bootstrap_iters, + cfg.confidence, + cfg.bootstrap_seed, + ); + println!( "speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}", ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload ); - let mut v = decide(ratio, cfg.threshold); - if cli.needs_human_review && v == Verdict::Accepted { - v = Verdict::NeedsHumanReview; - } - let fp = EnvFingerprint::collect(pin, "system-default"); - (v, Some(BenchEvidence { - baseline_median: bm, baseline_ci: (blo, bhi), - candidate_median: cm, candidate_ci: (clo, chi), - speedup_median: ratio.median, speedup_ci: (ratio.lo, ratio.hi), - env_fingerprint: serde_json::json!({ - "fingerprint": fp, - "workload": spec.bench.workload, - "target_commit": spec.source.commit, - "gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1", - }), - })) - }; + let mut v = decide(ratio, cfg.threshold); + if cli.needs_human_review && v == Verdict::Accepted { + v = Verdict::NeedsHumanReview; + } + let fp = EnvFingerprint::collect(pin, "system-default"); + ( + v, + Some(BenchEvidence { + baseline_median: bm, + baseline_ci: (blo, bhi), + candidate_median: cm, + candidate_ci: (clo, chi), + speedup_median: ratio.median, + speedup_ci: (ratio.lo, ratio.hi), + env_fingerprint: serde_json::json!({ + "fingerprint": fp, + "workload": spec.bench.workload, + "target_commit": spec.source.commit, + "gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1", + }), + }), + ) + }; let patch = match &cli.patch_file { Some(p) => std::fs::read_to_string(p)?, @@ -215,6 +249,10 @@ fn main() -> Result<()> { wall_time_s: started.elapsed().as_secs_f64(), }; Ledger::open(&root.join(&cli.db))?.record(&attempt)?; - println!("verdict: {} (ledger row {})", attempt.verdict.as_str(), attempt.run_id); + println!( + "verdict: {} (ledger row {})", + attempt.verdict.as_str(), + attempt.run_id + ); Ok(()) } diff --git a/docs/infra.md b/docs/infra.md new file mode 100644 index 0000000..0b9c150 --- /dev/null +++ b/docs/infra.md @@ -0,0 +1,37 @@ +# Bench infrastructure — what Phase 1 still needs from hardware + +The trust layer is code-complete for Phase 1 (calibration, gates, +verdict, agent IPC), but SPEC-grade *bench numbers* need hardware this +repo's CI cannot provide. Status by requirement: + +| Requirement (SPEC §3.1/§10) | GH-hosted runner | This repo's answer | +|---|---|---| +| No-network target execution | ✅ docker `--network=none` | `target-gates-no-net` CI job | +| Gates run mechanically | ✅ | `just gates` in CI | +| Environment noise measured | ✅ (measured, likely fails bars) | `runner-calibration` CI job, artifact per run | +| Profiling | ⚠️ software `cpu-clock` only (VM PMU usually absent) | `perf-probe` CI job reports what works | +| Pinned cores, governor, SMT/turbo control | ❌ shared VM | **bench-metal box (below)** | +| Separate-uid read-only trust layer | ❌ | bench-metal box setup | + +## The bench-metal box (human setup, ~15 min + account) + +Any dedicated bare-metal server works; the budget default is a Hetzner +AX-line dedicated server (other options: OVH/Equinix Metal/AWS +`*.metal`). Setup: + +1. Ubuntu LTS; `apt install linux-tools-$(uname -r) docker.io git build-essential`. +2. Measurement discipline: `cpupower frequency-set -g performance`; + disable SMT (`echo off > /sys/devices/system/cpu/smt/control`) and + turbo; reserve two cores via `isolcpus=` kernel arg; set + `pin_prefix` in `config/accept.toml` to those cores. +3. Register as a GitHub self-hosted runner with label `bench-metal` + (repo Settings → Actions → Runners). Run the runner as an + unprivileged user; make `crates/`, `config/`, `corpora/` owned by a + different user (this is the separate-uid enforcement from SPEC §10). +4. First act: `just calibrate results/calibration/bench-metal-01.json` + — the box is only trusted after its own A/A + injection evidence is + committed. Then flip verdict bench jobs to `runs-on: bench-metal`. + +Until that box exists, verdict-grade benches keep running wherever a +human has verified calibration evidence first (as done for this +container in `results/calibration/`). diff --git a/targets/fetch.sh b/targets/fetch.sh new file mode 100755 index 0000000..0f73a15 --- /dev/null +++ b/targets/fetch.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Reproduce a target workspace at its pinned commit (CI + fresh clones). +# Usage: targets/fetch.sh (run from repo root) +set -eu +name="$1" +toml="targets/$name/target.toml" +repo=$(grep -m1 '^repo = ' "$toml" | cut -d'"' -f2) +commit=$(grep -m1 '^commit = ' "$toml" | cut -d'"' -f2) +ws="targets/$name/workspace" +if [ ! -d "$ws/.git" ]; then + git clone "$repo" "$ws" +fi +git -C "$ws" fetch -q origin "$commit" 2>/dev/null || git -C "$ws" fetch -q origin +git -C "$ws" checkout -q "$commit" +# Pinned submodules (comrak's bench corpus lives in one). +grep -A0 '^"vendor/' "$toml" | while IFS= read -r line; do + sub=$(printf '%s' "$line" | cut -d'"' -f2) + surl=$(printf '%s' "$line" | sed 's/.*repo = "\([^"]*\)".*/\1/') + scommit=$(printf '%s' "$line" | sed 's/.*commit = "\([^"]*\)".*/\1/') + [ -d "$ws/$sub/.git" ] || git clone "$surl" "$ws/$sub" + git -C "$ws/$sub" fetch -q origin "$scommit" 2>/dev/null || true + git -C "$ws/$sub" checkout -q "$scommit" +done +echo "$name pinned at $commit"