mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
bench: GitHub Actions CI — trust-layer checks, no-net gates, runner calibration
Closes the CI-reachable slice of the Phase 1 infra gaps: - trust-layer job: cargo test + clippy -D warnings + fmt - target-gates-no-net: fetches pinned targets (new targets/fetch.sh), regenerates + verifies corpora, runs 'just gates tokei' mechanically, and executes the bench workload inside a --network=none container (SPEC 3.1/10 no-net requirement) - runner-calibration (informational): A/A + injection harness measures the shared runner's noise floor per run, evidence uploaded as an artifact — environments are judged empirically, not assumed - perf-probe (informational): reports which perf event sources work docs/infra.md maps each remaining hardware requirement (pinned cores, governor/SMT control, separate-uid trust layer) to the bench-metal self-hosted runner setup a human can complete in ~15 min. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
@@ -0,0 +1,87 @@
|
|||||||
|
name: ci
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
trust-layer:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- run: cargo test --workspace
|
||||||
|
- run: cargo clippy --workspace --all-targets -- -D warnings
|
||||||
|
- run: cargo fmt --all --check
|
||||||
|
|
||||||
|
# Phase 1: target gates run mechanically, and the bench workload
|
||||||
|
# executes inside a NO-NETWORK container (SPEC §3.1/§10). The runner
|
||||||
|
# is a shared VM — fine for gates; bench numbers for verdicts come
|
||||||
|
# from calibrated hardware, not this job.
|
||||||
|
target-gates-no-net:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- name: Fetch pinned targets
|
||||||
|
run: |
|
||||||
|
sh targets/fetch.sh comrak
|
||||||
|
sh targets/fetch.sh tokei
|
||||||
|
- name: Generate + verify pinned corpora
|
||||||
|
run: |
|
||||||
|
sh corpora/comrak/gen-corpus.sh
|
||||||
|
sh corpora/tokei/gen-corpus.sh
|
||||||
|
- name: Build tokei baseline
|
||||||
|
run: cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml
|
||||||
|
- name: Gates (corpus pin -> upstream tests -> golden replay)
|
||||||
|
run: cargo run -p diff-test -- tokei
|
||||||
|
- name: Bench workload inside no-network container
|
||||||
|
run: |
|
||||||
|
docker run --rm --network=none \
|
||||||
|
-v "$PWD:/w" -w /w \
|
||||||
|
debian:bookworm-slim \
|
||||||
|
sh -c "targets/tokei/workspace/target/release/tokei --sort code corpora/tokei/input | sha256sum"
|
||||||
|
|
||||||
|
# Informational: measure THIS runner's noise floor with the A/A +
|
||||||
|
# injection harness and archive the evidence. A shared VM failing the
|
||||||
|
# bars is expected — the artifact is the point (trust layer judges
|
||||||
|
# environments empirically; verdict benches belong on bench-metal).
|
||||||
|
runner-calibration:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
continue-on-error: true
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- name: Calibrate (reduced sessions, synthetic CPU-bound workload)
|
||||||
|
run: |
|
||||||
|
mkdir -p calib
|
||||||
|
cargo run -p bench-runner -- calibrate \
|
||||||
|
--cmd "head -c 3000000 /dev/urandom | sha256sum" \
|
||||||
|
--sessions 8 --out calib/gh-runner-calibration.json \
|
||||||
|
| tee calib/summary.txt
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
if: always()
|
||||||
|
with:
|
||||||
|
name: runner-calibration
|
||||||
|
path: calib/
|
||||||
|
|
||||||
|
# Informational: does perf work here, and with which event sources?
|
||||||
|
perf-probe:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
continue-on-error: true
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Probe perf availability
|
||||||
|
run: |
|
||||||
|
sudo apt-get -qq update && sudo apt-get -qq install -y linux-tools-common linux-tools-generic || true
|
||||||
|
PERF=$(ls /usr/lib/linux-tools/*/perf 2>/dev/null | head -1 || command -v perf || true)
|
||||||
|
echo "perf binary: ${PERF:-none}"
|
||||||
|
[ -n "$PERF" ] || exit 0
|
||||||
|
sudo sysctl -w kernel.perf_event_paranoid=1 || true
|
||||||
|
echo "--- software clock sampling:"
|
||||||
|
$PERF record -e cpu-clock -o /tmp/p.data -- sh -c 'i=0; while [ $i -lt 2000000 ]; do i=$((i+1)); done' && $PERF report -i /tmp/p.data --stdio | head -5 || echo "cpu-clock sampling unavailable"
|
||||||
|
echo "--- hardware cycle counter:"
|
||||||
|
$PERF stat -e cycles true 2>&1 | tail -3 || true
|
||||||
@@ -38,9 +38,7 @@ impl EnvFingerprint {
|
|||||||
Self {
|
Self {
|
||||||
kernel: read_trimmed("/proc/sys/kernel/osrelease"),
|
kernel: read_trimmed("/proc/sys/kernel/osrelease"),
|
||||||
cpu_model,
|
cpu_model,
|
||||||
governor: read_trimmed(
|
governor: read_trimmed("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"),
|
||||||
"/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
|
|
||||||
),
|
|
||||||
smt: read_trimmed("/sys/devices/system/cpu/smt/active"),
|
smt: read_trimmed("/sys/devices/system/cpu/smt/active"),
|
||||||
turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"),
|
turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"),
|
||||||
aslr: format!(
|
aslr: format!(
|
||||||
|
|||||||
@@ -34,10 +34,18 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn accept_requires_ci_lower_bound_not_point_estimate() {
|
fn accept_requires_ci_lower_bound_not_point_estimate() {
|
||||||
// 10% median speedup but the CI dips below threshold: rejected.
|
// 10% median speedup but the CI dips below threshold: rejected.
|
||||||
let noisy = RatioCi { median: 1.10, lo: 1.005, hi: 1.21 };
|
let noisy = RatioCi {
|
||||||
|
median: 1.10,
|
||||||
|
lo: 1.005,
|
||||||
|
hi: 1.21,
|
||||||
|
};
|
||||||
assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench);
|
assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench);
|
||||||
// 4% speedup with a tight CI: accepted.
|
// 4% speedup with a tight CI: accepted.
|
||||||
let tight = RatioCi { median: 1.04, lo: 1.025, hi: 1.055 };
|
let tight = RatioCi {
|
||||||
|
median: 1.04,
|
||||||
|
lo: 1.025,
|
||||||
|
hi: 1.055,
|
||||||
|
};
|
||||||
assert_eq!(decide(tight, 0.02), Verdict::Accepted);
|
assert_eq!(decide(tight, 0.02), Verdict::Accepted);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ use ledger::Verdict;
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")]
|
#[command(
|
||||||
|
name = "bench-runner",
|
||||||
|
about = "Interleaved A/B benchmark harness (hotpath trust layer)"
|
||||||
|
)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Path to accept.toml; built-in defaults if omitted.
|
/// Path to accept.toml; built-in defaults if omitted.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
@@ -68,23 +71,32 @@ fn calibrate(
|
|||||||
let seed = cfg.bootstrap_seed.wrapping_add(i as u64);
|
let seed = cfg.bootstrap_seed.wrapping_add(i as u64);
|
||||||
let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
|
let aa = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
|
||||||
let aa_ci = stats::bootstrap_ratio_ci(
|
let aa_ci = stats::bootstrap_ratio_ci(
|
||||||
&aa.baseline_s, &aa.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
|
&aa.baseline_s,
|
||||||
|
&aa.candidate_s,
|
||||||
|
cfg.bootstrap_iters,
|
||||||
|
cfg.confidence,
|
||||||
|
seed,
|
||||||
);
|
);
|
||||||
if decide(aa_ci, cfg.threshold) == Verdict::Accepted {
|
if decide(aa_ci, cfg.threshold) == Verdict::Accepted {
|
||||||
false_positives += 1;
|
false_positives += 1;
|
||||||
}
|
}
|
||||||
aa_sessions.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
|
aa_sessions
|
||||||
|
.push(serde_json::json!({"median": aa_ci.median, "lo": aa_ci.lo, "hi": aa_ci.hi}));
|
||||||
|
|
||||||
let inj =
|
let inj = exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
|
||||||
exec::run_interleaved(cmd, cmd, cfg.runs_per_side, cfg.warmup_runs, inject_s)?;
|
|
||||||
let inj_ci = stats::bootstrap_ratio_ci(
|
let inj_ci = stats::bootstrap_ratio_ci(
|
||||||
&inj.baseline_s, &inj.candidate_s, cfg.bootstrap_iters, cfg.confidence, seed,
|
&inj.baseline_s,
|
||||||
|
&inj.candidate_s,
|
||||||
|
cfg.bootstrap_iters,
|
||||||
|
cfg.confidence,
|
||||||
|
seed,
|
||||||
);
|
);
|
||||||
// Detected = the harness resolves a regression: CI entirely < 1.
|
// Detected = the harness resolves a regression: CI entirely < 1.
|
||||||
if inj_ci.hi < 1.0 {
|
if inj_ci.hi < 1.0 {
|
||||||
detections += 1;
|
detections += 1;
|
||||||
}
|
}
|
||||||
inj_sessions.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
|
inj_sessions
|
||||||
|
.push(serde_json::json!({"median": inj_ci.median, "lo": inj_ci.lo, "hi": inj_ci.hi}));
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}",
|
"session {}/{sessions}: A/A [{:.4},{:.4}] fp={false_positives}; inj [{:.4},{:.4}] det={detections}",
|
||||||
i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi
|
i + 1, aa_ci.lo, aa_ci.hi, inj_ci.lo, inj_ci.hi
|
||||||
@@ -127,9 +139,17 @@ fn main() -> Result<()> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (baseline, candidate, aa_mode) = match &cli.command {
|
let (baseline, candidate, aa_mode) = match &cli.command {
|
||||||
Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
|
Cmd::Compare {
|
||||||
|
baseline,
|
||||||
|
candidate,
|
||||||
|
} => (baseline.clone(), candidate.clone(), false),
|
||||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
||||||
Cmd::Calibrate { cmd, sessions, slowdown, out } => {
|
Cmd::Calibrate {
|
||||||
|
cmd,
|
||||||
|
sessions,
|
||||||
|
slowdown,
|
||||||
|
out,
|
||||||
|
} => {
|
||||||
return calibrate(&cfg, cmd, *sessions, *slowdown, out);
|
return calibrate(&cfg, cmd, *sessions, *slowdown, out);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -141,8 +161,13 @@ fn main() -> Result<()> {
|
|||||||
cfg.runs_per_side, cfg.warmup_runs
|
cfg.runs_per_side, cfg.warmup_runs
|
||||||
);
|
);
|
||||||
|
|
||||||
let samples =
|
let samples = exec::run_interleaved(
|
||||||
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs, 0.0)?;
|
&baseline,
|
||||||
|
&candidate,
|
||||||
|
cfg.runs_per_side,
|
||||||
|
cfg.warmup_runs,
|
||||||
|
0.0,
|
||||||
|
)?;
|
||||||
let ci = stats::bootstrap_ratio_ci(
|
let ci = stats::bootstrap_ratio_ci(
|
||||||
&samples.baseline_s,
|
&samples.baseline_s,
|
||||||
&samples.candidate_s,
|
&samples.candidate_s,
|
||||||
|
|||||||
@@ -54,7 +54,10 @@ pub fn bootstrap_ratio_ci(
|
|||||||
confidence: f64,
|
confidence: f64,
|
||||||
seed: u64,
|
seed: u64,
|
||||||
) -> RatioCi {
|
) -> RatioCi {
|
||||||
assert!(baseline.len() >= 2 && candidate.len() >= 2, "need >=2 runs per side");
|
assert!(
|
||||||
|
baseline.len() >= 2 && candidate.len() >= 2,
|
||||||
|
"need >=2 runs per side"
|
||||||
|
);
|
||||||
assert!((0.0..1.0).contains(&confidence));
|
assert!((0.0..1.0).contains(&confidence));
|
||||||
let mut rng = XorShift64::new(seed);
|
let mut rng = XorShift64::new(seed);
|
||||||
let mut ratios = Vec::with_capacity(iters);
|
let mut ratios = Vec::with_capacity(iters);
|
||||||
@@ -82,7 +85,12 @@ pub fn bootstrap_ratio_ci(
|
|||||||
|
|
||||||
/// Percentile-method bootstrap CI of a single sample's median, for
|
/// Percentile-method bootstrap CI of a single sample's median, for
|
||||||
/// per-side evidence in ledger rows.
|
/// per-side evidence in ledger rows.
|
||||||
pub fn bootstrap_median_ci(sample: &[f64], iters: usize, confidence: f64, seed: u64) -> (f64, f64, f64) {
|
pub fn bootstrap_median_ci(
|
||||||
|
sample: &[f64],
|
||||||
|
iters: usize,
|
||||||
|
confidence: f64,
|
||||||
|
seed: u64,
|
||||||
|
) -> (f64, f64, f64) {
|
||||||
assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence));
|
assert!(sample.len() >= 2 && (0.0..1.0).contains(&confidence));
|
||||||
let mut rng = XorShift64::new(seed);
|
let mut rng = XorShift64::new(seed);
|
||||||
let mut medians = Vec::with_capacity(iters);
|
let mut medians = Vec::with_capacity(iters);
|
||||||
@@ -106,7 +114,9 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn median_ci_brackets_median() {
|
fn median_ci_brackets_median() {
|
||||||
let s: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
|
let s: Vec<f64> = (0..40)
|
||||||
|
.map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64)
|
||||||
|
.collect();
|
||||||
let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9);
|
let (m, lo, hi) = bootstrap_median_ci(&s, 1000, 0.95, 9);
|
||||||
assert!(lo <= m && m <= hi);
|
assert!(lo <= m && m <= hi);
|
||||||
}
|
}
|
||||||
@@ -120,15 +130,22 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn aa_ci_straddles_one() {
|
fn aa_ci_straddles_one() {
|
||||||
// Same distribution both sides: the CI must contain 1.0.
|
// Same distribution both sides: the CI must contain 1.0.
|
||||||
let sample: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
|
let sample: Vec<f64> = (0..40)
|
||||||
|
.map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64)
|
||||||
|
.collect();
|
||||||
let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42);
|
let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42);
|
||||||
assert!(ci.lo <= 1.0 && ci.hi >= 1.0, "A/A CI must straddle 1.0, got {ci:?}");
|
assert!(
|
||||||
|
ci.lo <= 1.0 && ci.hi >= 1.0,
|
||||||
|
"A/A CI must straddle 1.0, got {ci:?}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn detects_injected_regression() {
|
fn detects_injected_regression() {
|
||||||
// Candidate 5% slower with mild noise: speedup CI entirely < 1.
|
// Candidate 5% slower with mild noise: speedup CI entirely < 1.
|
||||||
let base: Vec<f64> = (0..40).map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64).collect();
|
let base: Vec<f64> = (0..40)
|
||||||
|
.map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64)
|
||||||
|
.collect();
|
||||||
let cand: Vec<f64> = base.iter().map(|t| t * 1.05).collect();
|
let cand: Vec<f64> = base.iter().map(|t| t * 1.05).collect();
|
||||||
let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42);
|
let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42);
|
||||||
assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}");
|
assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}");
|
||||||
|
|||||||
@@ -9,8 +9,8 @@
|
|||||||
|
|
||||||
pub mod orchestrate;
|
pub mod orchestrate;
|
||||||
pub mod pin;
|
pub mod pin;
|
||||||
pub mod target;
|
|
||||||
pub mod policy;
|
pub mod policy;
|
||||||
|
pub mod target;
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
@@ -35,9 +35,10 @@ pub enum GateLayer {
|
|||||||
|
|
||||||
/// All layers must pass; any failure is an auto-reject (SPEC §8).
|
/// All layers must pass; any failure is an auto-reject (SPEC §8).
|
||||||
pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool {
|
pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool {
|
||||||
results.iter().all(|(_, o)| {
|
results
|
||||||
matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. })
|
.iter()
|
||||||
}) && results
|
.all(|(_, o)| matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. }))
|
||||||
|
&& results
|
||||||
.iter()
|
.iter()
|
||||||
.any(|(_, o)| matches!(o, GateOutcome::Passed))
|
.any(|(_, o)| matches!(o, GateOutcome::Passed))
|
||||||
}
|
}
|
||||||
@@ -52,7 +53,9 @@ mod tests {
|
|||||||
(GateLayer::UpstreamTests, GateOutcome::Passed),
|
(GateLayer::UpstreamTests, GateOutcome::Passed),
|
||||||
(
|
(
|
||||||
GateLayer::GoldenReplay,
|
GateLayer::GoldenReplay,
|
||||||
GateOutcome::Failed { detail: "stdout diverged".into() },
|
GateOutcome::Failed {
|
||||||
|
detail: "stdout diverged".into(),
|
||||||
|
},
|
||||||
),
|
),
|
||||||
];
|
];
|
||||||
assert!(!all_passed(&results));
|
assert!(!all_passed(&results));
|
||||||
@@ -62,7 +65,9 @@ mod tests {
|
|||||||
fn all_skipped_is_not_a_pass() {
|
fn all_skipped_is_not_a_pass() {
|
||||||
let results = vec![(
|
let results = vec![(
|
||||||
GateLayer::Sanitizers,
|
GateLayer::Sanitizers,
|
||||||
GateOutcome::Skipped { reason: "n/a".into() },
|
GateOutcome::Skipped {
|
||||||
|
reason: "n/a".into(),
|
||||||
|
},
|
||||||
)];
|
)];
|
||||||
assert!(!all_passed(&results));
|
assert!(!all_passed(&results));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,7 +43,10 @@ pub fn run_core_gates(
|
|||||||
GateOutcome::Passed
|
GateOutcome::Passed
|
||||||
} else {
|
} else {
|
||||||
GateOutcome::Failed {
|
GateOutcome::Failed {
|
||||||
detail: String::from_utf8_lossy(&t.stderr).chars().take(2000).collect(),
|
detail: String::from_utf8_lossy(&t.stderr)
|
||||||
|
.chars()
|
||||||
|
.take(2000)
|
||||||
|
.collect(),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
@@ -57,7 +60,10 @@ pub fn run_core_gates(
|
|||||||
GateOutcome::Passed
|
GateOutcome::Passed
|
||||||
} else {
|
} else {
|
||||||
GateOutcome::Failed {
|
GateOutcome::Failed {
|
||||||
detail: format!("exit={:?} expected={expected} actual={actual}", g.status.code()),
|
detail: format!(
|
||||||
|
"exit={:?} expected={expected} actual={actual}",
|
||||||
|
g.status.code()
|
||||||
|
),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -67,7 +67,10 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
EquivalencePolicy::load(&dir).unwrap(),
|
EquivalencePolicy::load(&dir).unwrap(),
|
||||||
EquivalencePolicy::FpTolerance { abs: 1e-9, rel: 1e-6 }
|
EquivalencePolicy::FpTolerance {
|
||||||
|
abs: 1e-9,
|
||||||
|
rel: 1e-6
|
||||||
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+49
-15
@@ -41,7 +41,10 @@ fn field<'a>(req: &'a Value, key: &str) -> Result<&'a str> {
|
|||||||
/// A target name must be a plain directory name — no separators, no dots.
|
/// A target name must be a plain directory name — no separators, no dots.
|
||||||
fn check_target(name: &str) -> Result<()> {
|
fn check_target(name: &str) -> Result<()> {
|
||||||
ensure!(
|
ensure!(
|
||||||
!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
|
!name.is_empty()
|
||||||
|
&& name
|
||||||
|
.chars()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
|
||||||
"invalid target name"
|
"invalid target name"
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -88,7 +91,11 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
|
|||||||
let dir = root.join("playbook");
|
let dir = root.join("playbook");
|
||||||
let entry = std::fs::read_dir(&dir)?
|
let entry = std::fs::read_dir(&dir)?
|
||||||
.filter_map(|e| e.ok())
|
.filter_map(|e| e.ok())
|
||||||
.find(|e| e.file_name().to_string_lossy().starts_with(&format!("{class:02}-")))
|
.find(|e| {
|
||||||
|
e.file_name()
|
||||||
|
.to_string_lossy()
|
||||||
|
.starts_with(&format!("{class:02}-"))
|
||||||
|
})
|
||||||
.ok_or_else(|| anyhow!("no playbook file for class {class}"))?;
|
.ok_or_else(|| anyhow!("no playbook file for class {class}"))?;
|
||||||
Ok(json!({"markdown": std::fs::read_to_string(entry.path())?}))
|
Ok(json!({"markdown": std::fs::read_to_string(entry.path())?}))
|
||||||
}
|
}
|
||||||
@@ -109,10 +116,16 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
|
|||||||
// relative path (the git -C below roots them in the target
|
// relative path (the git -C below roots them in the target
|
||||||
// workspace; nothing outside it is reachable).
|
// workspace; nothing outside it is reachable).
|
||||||
for l in diff.lines() {
|
for l in diff.lines() {
|
||||||
if let Some(p) = l.strip_prefix("--- a/").or_else(|| l.strip_prefix("+++ b/")) {
|
if let Some(p) = l
|
||||||
|
.strip_prefix("--- a/")
|
||||||
|
.or_else(|| l.strip_prefix("+++ b/"))
|
||||||
|
{
|
||||||
check_rel_path(p.trim())?;
|
check_rel_path(p.trim())?;
|
||||||
}
|
}
|
||||||
ensure!(!l.starts_with("--- /") && !l.starts_with("+++ /"), "absolute diff paths forbidden");
|
ensure!(
|
||||||
|
!l.starts_with("--- /") && !l.starts_with("+++ /"),
|
||||||
|
"absolute diff paths forbidden"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let ws = root.join(format!("targets/{t}/workspace"));
|
let ws = root.join(format!("targets/{t}/workspace"));
|
||||||
let mut check = Command::new("git")
|
let mut check = Command::new("git")
|
||||||
@@ -132,13 +145,18 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
|
|||||||
std::fs::create_dir_all(&pending)?;
|
std::fs::create_dir_all(&pending)?;
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
pending.join(format!("{patch_id}.json")),
|
pending.join(format!("{patch_id}.json")),
|
||||||
serde_json::to_string_pretty(&json!({"target": t, "hypothesis": hypothesis, "diff": diff}))?,
|
serde_json::to_string_pretty(
|
||||||
|
&json!({"target": t, "hypothesis": hypothesis, "diff": diff}),
|
||||||
|
)?,
|
||||||
)?;
|
)?;
|
||||||
Ok(json!({"patch_id": patch_id}))
|
Ok(json!({"patch_id": patch_id}))
|
||||||
}
|
}
|
||||||
"run_verdict" => {
|
"run_verdict" => {
|
||||||
let patch_id = field(&req, "patch_id")?;
|
let patch_id = field(&req, "patch_id")?;
|
||||||
ensure!(patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12, "bad patch id");
|
ensure!(
|
||||||
|
patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12,
|
||||||
|
"bad patch id"
|
||||||
|
);
|
||||||
let run_id = field(&req, "run_id")?;
|
let run_id = field(&req, "run_id")?;
|
||||||
let class = field(&req, "playbook_class")?;
|
let class = field(&req, "playbook_class")?;
|
||||||
let hotspot = field(&req, "hotspot")?;
|
let hotspot = field(&req, "hotspot")?;
|
||||||
@@ -151,24 +169,40 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
|
|||||||
// isolated dir, then hand off to the verdict pipeline.
|
// isolated dir, then hand off to the verdict pipeline.
|
||||||
let cand_dir = root.join(format!("targets/{t}/candidate-{patch_id}"));
|
let cand_dir = root.join(format!("targets/{t}/candidate-{patch_id}"));
|
||||||
let build = Command::new("sh")
|
let build = Command::new("sh")
|
||||||
.arg("-c").arg(&spec.build.baseline)
|
.arg("-c")
|
||||||
|
.arg(&spec.build.baseline)
|
||||||
.current_dir(root)
|
.current_dir(root)
|
||||||
.env("CARGO_TARGET_DIR", &cand_dir)
|
.env("CARGO_TARGET_DIR", &cand_dir)
|
||||||
.status()?;
|
.status()?;
|
||||||
ensure!(build.success(), "candidate build failed");
|
ensure!(build.success(), "candidate build failed");
|
||||||
let bin = Path::new(&spec.build.binary).file_name().unwrap().to_str().unwrap();
|
let bin = Path::new(&spec.build.binary)
|
||||||
|
.file_name()
|
||||||
|
.unwrap()
|
||||||
|
.to_str()
|
||||||
|
.unwrap();
|
||||||
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
||||||
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
|
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
|
||||||
let out = Command::new("cargo")
|
let out = Command::new("cargo")
|
||||||
.args([
|
.args([
|
||||||
"run", "-q", "-p", "verdict", "--", t,
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"verdict",
|
||||||
|
"--",
|
||||||
|
t,
|
||||||
"--rebuild-baseline",
|
"--rebuild-baseline",
|
||||||
"--candidate-bin", &format!("targets/{t}/candidate-{patch_id}/release/{bin}"),
|
"--candidate-bin",
|
||||||
"--run-id", run_id,
|
&format!("targets/{t}/candidate-{patch_id}/release/{bin}"),
|
||||||
"--playbook-class", class,
|
"--run-id",
|
||||||
"--hypothesis", pending["hypothesis"].as_str().unwrap(),
|
run_id,
|
||||||
"--hotspot", hotspot,
|
"--playbook-class",
|
||||||
"--patch-file", diff_path.to_str().unwrap(),
|
class,
|
||||||
|
"--hypothesis",
|
||||||
|
pending["hypothesis"].as_str().unwrap(),
|
||||||
|
"--hotspot",
|
||||||
|
hotspot,
|
||||||
|
"--patch-file",
|
||||||
|
diff_path.to_str().unwrap(),
|
||||||
])
|
])
|
||||||
.current_dir(root)
|
.current_dir(root)
|
||||||
.output()?;
|
.output()?;
|
||||||
|
|||||||
+56
-18
@@ -8,12 +8,17 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use diff_test::{all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome};
|
use diff_test::{
|
||||||
|
all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome,
|
||||||
|
};
|
||||||
use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict};
|
use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(name = "verdict", about = "gates + bench + ledger row for one optimization attempt")]
|
#[command(
|
||||||
|
name = "verdict",
|
||||||
|
about = "gates + bench + ledger row for one optimization attempt"
|
||||||
|
)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Target name under targets/.
|
/// Target name under targets/.
|
||||||
target: String,
|
target: String,
|
||||||
@@ -78,7 +83,10 @@ fn rebuild_pristine_baseline(
|
|||||||
.arg("-c")
|
.arg("-c")
|
||||||
.arg(&spec.build.baseline)
|
.arg(&spec.build.baseline)
|
||||||
.current_dir(root)
|
.current_dir(root)
|
||||||
.env("CARGO_TARGET_DIR", root.join(format!("targets/{target}/baseline")))
|
.env(
|
||||||
|
"CARGO_TARGET_DIR",
|
||||||
|
root.join(format!("targets/{target}/baseline")),
|
||||||
|
)
|
||||||
.status();
|
.status();
|
||||||
if dirty {
|
if dirty {
|
||||||
anyhow::ensure!(
|
anyhow::ensure!(
|
||||||
@@ -95,7 +103,10 @@ fn rebuild_pristine_baseline(
|
|||||||
.and_then(|n| n.to_str())
|
.and_then(|n| n.to_str())
|
||||||
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
||||||
let baseline = format!("targets/{target}/baseline/release/{bin_name}");
|
let baseline = format!("targets/{target}/baseline/release/{bin_name}");
|
||||||
anyhow::ensure!(root.join(&baseline).exists(), "baseline binary missing: {baseline}");
|
anyhow::ensure!(
|
||||||
|
root.join(&baseline).exists(),
|
||||||
|
"baseline binary missing: {baseline}"
|
||||||
|
);
|
||||||
println!("baseline rebuilt from pristine checkout: {baseline}");
|
println!("baseline rebuilt from pristine checkout: {baseline}");
|
||||||
Ok(baseline)
|
Ok(baseline)
|
||||||
}
|
}
|
||||||
@@ -147,10 +158,15 @@ fn main() -> Result<()> {
|
|||||||
let pin = &cfg.pin_prefix;
|
let pin = &cfg.pin_prefix;
|
||||||
let wrap = |bin: &str| {
|
let wrap = |bin: &str| {
|
||||||
let cmd = spec.bench.command.replace("{binary}", bin);
|
let cmd = spec.bench.command.replace("{binary}", bin);
|
||||||
if pin.is_empty() { format!("sh -c \"{cmd}\"") } else { format!("{pin} sh -c \"{cmd}\"") }
|
if pin.is_empty() {
|
||||||
|
format!("sh -c \"{cmd}\"")
|
||||||
|
} else {
|
||||||
|
format!("{pin} sh -c \"{cmd}\"")
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let (verdict, bench) = if !all_passed(&gates) {
|
let (verdict, bench) =
|
||||||
|
if !all_passed(&gates) {
|
||||||
println!("verdict: rejected-gate (bench skipped)");
|
println!("verdict: rejected-gate (bench skipped)");
|
||||||
(Verdict::RejectedGate, None)
|
(Verdict::RejectedGate, None)
|
||||||
} else {
|
} else {
|
||||||
@@ -159,18 +175,30 @@ fn main() -> Result<()> {
|
|||||||
cfg.runs_per_side, cfg.warmup_runs, pin
|
cfg.runs_per_side, cfg.warmup_runs, pin
|
||||||
);
|
);
|
||||||
let samples = exec::run_interleaved(
|
let samples = exec::run_interleaved(
|
||||||
&wrap(&baseline_bin), &wrap(&cli.candidate_bin),
|
&wrap(&baseline_bin),
|
||||||
cfg.runs_per_side, cfg.warmup_runs, 0.0,
|
&wrap(&cli.candidate_bin),
|
||||||
|
cfg.runs_per_side,
|
||||||
|
cfg.warmup_runs,
|
||||||
|
0.0,
|
||||||
)?;
|
)?;
|
||||||
let ratio = stats::bootstrap_ratio_ci(
|
let ratio = stats::bootstrap_ratio_ci(
|
||||||
&samples.baseline_s, &samples.candidate_s,
|
&samples.baseline_s,
|
||||||
cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
|
&samples.candidate_s,
|
||||||
|
cfg.bootstrap_iters,
|
||||||
|
cfg.confidence,
|
||||||
|
cfg.bootstrap_seed,
|
||||||
);
|
);
|
||||||
let (bm, blo, bhi) = stats::bootstrap_median_ci(
|
let (bm, blo, bhi) = stats::bootstrap_median_ci(
|
||||||
&samples.baseline_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
|
&samples.baseline_s,
|
||||||
|
cfg.bootstrap_iters,
|
||||||
|
cfg.confidence,
|
||||||
|
cfg.bootstrap_seed,
|
||||||
);
|
);
|
||||||
let (cm, clo, chi) = stats::bootstrap_median_ci(
|
let (cm, clo, chi) = stats::bootstrap_median_ci(
|
||||||
&samples.candidate_s, cfg.bootstrap_iters, cfg.confidence, cfg.bootstrap_seed,
|
&samples.candidate_s,
|
||||||
|
cfg.bootstrap_iters,
|
||||||
|
cfg.confidence,
|
||||||
|
cfg.bootstrap_seed,
|
||||||
);
|
);
|
||||||
println!(
|
println!(
|
||||||
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
|
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
|
||||||
@@ -181,17 +209,23 @@ fn main() -> Result<()> {
|
|||||||
v = Verdict::NeedsHumanReview;
|
v = Verdict::NeedsHumanReview;
|
||||||
}
|
}
|
||||||
let fp = EnvFingerprint::collect(pin, "system-default");
|
let fp = EnvFingerprint::collect(pin, "system-default");
|
||||||
(v, Some(BenchEvidence {
|
(
|
||||||
baseline_median: bm, baseline_ci: (blo, bhi),
|
v,
|
||||||
candidate_median: cm, candidate_ci: (clo, chi),
|
Some(BenchEvidence {
|
||||||
speedup_median: ratio.median, speedup_ci: (ratio.lo, ratio.hi),
|
baseline_median: bm,
|
||||||
|
baseline_ci: (blo, bhi),
|
||||||
|
candidate_median: cm,
|
||||||
|
candidate_ci: (clo, chi),
|
||||||
|
speedup_median: ratio.median,
|
||||||
|
speedup_ci: (ratio.lo, ratio.hi),
|
||||||
env_fingerprint: serde_json::json!({
|
env_fingerprint: serde_json::json!({
|
||||||
"fingerprint": fp,
|
"fingerprint": fp,
|
||||||
"workload": spec.bench.workload,
|
"workload": spec.bench.workload,
|
||||||
"target_commit": spec.source.commit,
|
"target_commit": spec.source.commit,
|
||||||
"gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
|
"gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
|
||||||
}),
|
}),
|
||||||
}))
|
}),
|
||||||
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
let patch = match &cli.patch_file {
|
let patch = match &cli.patch_file {
|
||||||
@@ -215,6 +249,10 @@ fn main() -> Result<()> {
|
|||||||
wall_time_s: started.elapsed().as_secs_f64(),
|
wall_time_s: started.elapsed().as_secs_f64(),
|
||||||
};
|
};
|
||||||
Ledger::open(&root.join(&cli.db))?.record(&attempt)?;
|
Ledger::open(&root.join(&cli.db))?.record(&attempt)?;
|
||||||
println!("verdict: {} (ledger row {})", attempt.verdict.as_str(), attempt.run_id);
|
println!(
|
||||||
|
"verdict: {} (ledger row {})",
|
||||||
|
attempt.verdict.as_str(),
|
||||||
|
attempt.run_id
|
||||||
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Bench infrastructure — what Phase 1 still needs from hardware
|
||||||
|
|
||||||
|
The trust layer is code-complete for Phase 1 (calibration, gates,
|
||||||
|
verdict, agent IPC), but SPEC-grade *bench numbers* need hardware this
|
||||||
|
repo's CI cannot provide. Status by requirement:
|
||||||
|
|
||||||
|
| Requirement (SPEC §3.1/§10) | GH-hosted runner | This repo's answer |
|
||||||
|
|---|---|---|
|
||||||
|
| No-network target execution | ✅ docker `--network=none` | `target-gates-no-net` CI job |
|
||||||
|
| Gates run mechanically | ✅ | `just gates` in CI |
|
||||||
|
| Environment noise measured | ✅ (measured, likely fails bars) | `runner-calibration` CI job, artifact per run |
|
||||||
|
| Profiling | ⚠️ software `cpu-clock` only (VM PMU usually absent) | `perf-probe` CI job reports what works |
|
||||||
|
| Pinned cores, governor, SMT/turbo control | ❌ shared VM | **bench-metal box (below)** |
|
||||||
|
| Separate-uid read-only trust layer | ❌ | bench-metal box setup |
|
||||||
|
|
||||||
|
## The bench-metal box (human setup, ~15 min + account)
|
||||||
|
|
||||||
|
Any dedicated bare-metal server works; the budget default is a Hetzner
|
||||||
|
AX-line dedicated server (other options: OVH/Equinix Metal/AWS
|
||||||
|
`*.metal`). Setup:
|
||||||
|
|
||||||
|
1. Ubuntu LTS; `apt install linux-tools-$(uname -r) docker.io git build-essential`.
|
||||||
|
2. Measurement discipline: `cpupower frequency-set -g performance`;
|
||||||
|
disable SMT (`echo off > /sys/devices/system/cpu/smt/control`) and
|
||||||
|
turbo; reserve two cores via `isolcpus=` kernel arg; set
|
||||||
|
`pin_prefix` in `config/accept.toml` to those cores.
|
||||||
|
3. Register as a GitHub self-hosted runner with label `bench-metal`
|
||||||
|
(repo Settings → Actions → Runners). Run the runner as an
|
||||||
|
unprivileged user; make `crates/`, `config/`, `corpora/` owned by a
|
||||||
|
different user (this is the separate-uid enforcement from SPEC §10).
|
||||||
|
4. First act: `just calibrate <real workload> results/calibration/bench-metal-01.json`
|
||||||
|
— the box is only trusted after its own A/A + injection evidence is
|
||||||
|
committed. Then flip verdict bench jobs to `runs-on: bench-metal`.
|
||||||
|
|
||||||
|
Until that box exists, verdict-grade benches keep running wherever a
|
||||||
|
human has verified calibration evidence first (as done for this
|
||||||
|
container in `results/calibration/`).
|
||||||
Executable
+24
@@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Reproduce a target workspace at its pinned commit (CI + fresh clones).
|
||||||
|
# Usage: targets/fetch.sh <name> (run from repo root)
|
||||||
|
set -eu
|
||||||
|
name="$1"
|
||||||
|
toml="targets/$name/target.toml"
|
||||||
|
repo=$(grep -m1 '^repo = ' "$toml" | cut -d'"' -f2)
|
||||||
|
commit=$(grep -m1 '^commit = ' "$toml" | cut -d'"' -f2)
|
||||||
|
ws="targets/$name/workspace"
|
||||||
|
if [ ! -d "$ws/.git" ]; then
|
||||||
|
git clone "$repo" "$ws"
|
||||||
|
fi
|
||||||
|
git -C "$ws" fetch -q origin "$commit" 2>/dev/null || git -C "$ws" fetch -q origin
|
||||||
|
git -C "$ws" checkout -q "$commit"
|
||||||
|
# Pinned submodules (comrak's bench corpus lives in one).
|
||||||
|
grep -A0 '^"vendor/' "$toml" | while IFS= read -r line; do
|
||||||
|
sub=$(printf '%s' "$line" | cut -d'"' -f2)
|
||||||
|
surl=$(printf '%s' "$line" | sed 's/.*repo = "\([^"]*\)".*/\1/')
|
||||||
|
scommit=$(printf '%s' "$line" | sed 's/.*commit = "\([^"]*\)".*/\1/')
|
||||||
|
[ -d "$ws/$sub/.git" ] || git clone "$surl" "$ws/$sub"
|
||||||
|
git -C "$ws/$sub" fetch -q origin "$scommit" 2>/dev/null || true
|
||||||
|
git -C "$ws/$sub" checkout -q "$scommit"
|
||||||
|
done
|
||||||
|
echo "$name pinned at $commit"
|
||||||
Reference in New Issue
Block a user