mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
gates: close the audit findings — enforce the gates the docs claimed
An adversarial audit found the enforcement story ahead of the code: gates that were documented as hard gates were advisory, manual, or absent, and several human-facing claims outran the machine record. This makes the claims true. High findings - Differential fuzz was documentation, not enforcement: verdict hardcoded fuzz_iters=0 and diff-test hardcoded the gate to Skipped, while the case studies claimed 10k-iteration fuzz on the same wins. Fuzz is now a real pipeline gate (strict FUZZ-RESULT contract, executed count recorded), and it is inherently baseline-vs-candidate, so it runs on the accept path and reports Skipped in the standalone `just gates` flow. New hard rule: a machine `accepted` verdict is impossible unless the fuzz gate actually passed — otherwise it caps at needs-human-review. - The agent's system prompt claimed FP/concurrency/unsafe patches were "routed to human review automatically". No such mechanism existed. Added a conservative lexical risk classifier (crates/verdict/src/risk.rs) that forces needs-human-review on any would-be accept carrying those signals; fp-tolerance equivalence always signals. - The local verdict path ran agent-influenced code unsandboxed. It now runs under scripts/no-net.sh (network namespace) and fails closed. Upstream test suites are hash-verified against corpora/<t>/TESTSUITE.sha256 when present (scripts/pin-testsuite.sh generates them). Also fixed, found while fixing the above - The pipeline script's `set -e` was inert (brace group as the left operand of `&&`), so a failing candidate build did not abort the run and verdict could bench a stale binary. Now a scoped subshell whose status is branched on at top level. - A wrapper that exits before exec'ing (no-net.sh refusing, bad wrapper path) stranded RUNNING.lock and left read_verdict polling forever. Failure accounting now lives outside the wrapper. - harnessd's patch allowlist ignored git rename/copy headers and --no-prefix diffs; the rename-escape case is now a committed regression test. - The workspace did not compile on Windows (ungated Unix APIs). cfg-gated; 71 portable tests now run on both platforms. - pin-check regenerated the corpus manifest instead of verifying it; it is now verify-only, with a deliberate `just pin-corpus` to re-pin. - Max RSS is captured per run on Unix; PMU counters are opt-in diagnostics (compare --perf-stat), never part of the accept decision. Docs now match the code, including what is NOT yet enforced: no target ships a TESTSUITE.sha256 pin, no target declares [build].tsan, and matmul is not wired to the fp-tolerance gate — each is stated as mechanized-but-inert rather than done. Corrected the stale "33 ledger rows" (34), the playbook's class-3 cJSON win (rejected), adversarial-review.md's sanitizer-separation claim (five accepted rows carry sanitizers_clean=false), the unsupported "40,000 requests" figure, and the blanket "historical rows record fuzz_iters=0" (8 of 10; phase0-tokei-002/-003 record 4,332). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -10,7 +10,9 @@ repository.workspace = true
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
ledger = { path = "../ledger" }
|
||||
libc = "0.2"
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
@@ -23,6 +23,13 @@ pub struct AcceptConfig {
|
||||
/// both sides identically.
|
||||
#[serde(default)]
|
||||
pub pin_prefix: String,
|
||||
/// Differential-fuzz iterations requested per gate run (SPEC §3.2).
|
||||
#[serde(default = "default_fuzz_iters")]
|
||||
pub fuzz_iters: u64,
|
||||
}
|
||||
|
||||
fn default_fuzz_iters() -> u64 {
|
||||
10_000
|
||||
}
|
||||
|
||||
impl Default for AcceptConfig {
|
||||
@@ -35,6 +42,7 @@ impl Default for AcceptConfig {
|
||||
confidence: 0.95,
|
||||
bootstrap_seed: 0x707E_17E5,
|
||||
pin_prefix: String::new(),
|
||||
fuzz_iters: default_fuzz_iters(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,3 +60,31 @@ impl AcceptConfig {
|
||||
Ok(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const MINIMAL: &str = "threshold = 0.02\nruns_per_side = 30\nwarmup_runs = 3\n\
|
||||
bootstrap_iters = 10000\nconfidence = 0.95\nbootstrap_seed = 7\n";
|
||||
|
||||
#[test]
|
||||
fn fuzz_iters_defaults_when_absent() {
|
||||
let cfg: AcceptConfig = toml::from_str(MINIMAL).unwrap();
|
||||
assert_eq!(cfg.fuzz_iters, 10_000);
|
||||
assert_eq!(AcceptConfig::default().fuzz_iters, 10_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_iters_overridable() {
|
||||
let raw = format!("{MINIMAL}fuzz_iters = 500\n");
|
||||
let cfg: AcceptConfig = toml::from_str(&raw).unwrap();
|
||||
assert_eq!(cfg.fuzz_iters, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_still_rejected() {
|
||||
let raw = format!("{MINIMAL}fuz_iters = 500\n");
|
||||
assert!(toml::from_str::<AcceptConfig>(&raw).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Opt-in `perf stat` PMU counter capture. Diagnostics only — counters
|
||||
//! never feed the accept decision (wall time does, SPEC §3.1). Each
|
||||
//! timed run is wrapped `perf stat -x, -e <events> -o <tmpfile> -- sh -c
|
||||
//! <cmd>`; the CSV file is parsed after every run and per-side medians
|
||||
//! are printed with the standard caveats. Unix-only at runtime; the
|
||||
//! parser is portable and unit-tested.
|
||||
|
||||
use crate::exec::MeasuredSamples;
|
||||
use crate::schedule::{interleaved, Side};
|
||||
use crate::stats;
|
||||
use anyhow::Context;
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
pub const PERF_EVENTS: [&str; 4] = ["cycles", "instructions", "cache-misses", "branch-misses"];
|
||||
|
||||
/// One side's counter samples: `samples[i]` holds one value per measured
|
||||
/// run for `PERF_EVENTS[i]`; runs where perf reported `<not counted>` or
|
||||
/// `<not supported>` contribute nothing.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PerfSide {
|
||||
pub samples: [Vec<u64>; 4],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PerfReport {
|
||||
pub baseline: PerfSide,
|
||||
pub candidate: PerfSide,
|
||||
}
|
||||
|
||||
/// Parse `perf stat -x,` CSV: field 0 is the counter value, field 2 the
|
||||
/// event name (perf may append a modifier, e.g. `cycles:u`). Comment
|
||||
/// lines, blank lines, and non-numeric values (`<not counted>`,
|
||||
/// `<not supported>`) are skipped. Returns (event, count) in file order.
|
||||
pub fn parse_perf_csv(text: &str) -> Vec<(String, u64)> {
|
||||
let mut out = Vec::new();
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let mut fields = line.split(',');
|
||||
let value = fields.next().unwrap_or("");
|
||||
let _unit = fields.next();
|
||||
let Some(event) = fields.next() else {
|
||||
continue;
|
||||
};
|
||||
let Ok(v) = value.trim().parse::<u64>() else {
|
||||
continue;
|
||||
};
|
||||
out.push((event.trim().to_string(), v));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Index into PERF_EVENTS, tolerating a perf modifier suffix
|
||||
/// (`cycles:u` when perf_event_paranoid restricts kernel counting).
|
||||
fn event_index(event: &str) -> Option<usize> {
|
||||
PERF_EVENTS.iter().position(|name| {
|
||||
event == *name || (event.starts_with(name) && event[name.len()..].starts_with(':'))
|
||||
})
|
||||
}
|
||||
|
||||
impl PerfSide {
|
||||
fn absorb(&mut self, parsed: &[(String, u64)]) {
|
||||
for (event, value) in parsed {
|
||||
if let Some(i) = event_index(event) {
|
||||
self.samples[i].push(*value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PerfReport {
|
||||
/// Per-side medians, each with its command (workload) and run count.
|
||||
pub fn print(&self, baseline_cmd: &str, candidate_cmd: &str) {
|
||||
println!("perf-stat per-side medians:");
|
||||
for (label, cmd, side) in [
|
||||
("baseline ", baseline_cmd, &self.baseline),
|
||||
("candidate", candidate_cmd, &self.candidate),
|
||||
] {
|
||||
let cols: Vec<String> = PERF_EVENTS
|
||||
.iter()
|
||||
.zip(&side.samples)
|
||||
.map(|(name, xs)| {
|
||||
if xs.is_empty() {
|
||||
format!("{name}=n/a")
|
||||
} else {
|
||||
let v: Vec<f64> = xs.iter().map(|&x| x as f64).collect();
|
||||
format!(
|
||||
"{name}={:.0} (median of {} runs)",
|
||||
stats::median(&v),
|
||||
xs.len()
|
||||
)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
println!(" {label} `{cmd}`: {}", cols.join(", "));
|
||||
}
|
||||
println!(
|
||||
" note: PMU counters unavailable in most VMs — flag is opt-in diagnostics, \
|
||||
not part of the accept decision"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn time_once_perf(shell_cmd: &str, out_file: &Path) -> anyhow::Result<(f64, Vec<(String, u64)>)> {
|
||||
let start = Instant::now();
|
||||
let status = Command::new("perf")
|
||||
.args(["stat", "-x,", "-e", &PERF_EVENTS.join(",")])
|
||||
.arg("-o")
|
||||
.arg(out_file)
|
||||
.args(["--", "sh", "-c", shell_cmd])
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.with_context(|| format!("failed to spawn perf stat for: {shell_cmd}"))?;
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
anyhow::ensure!(
|
||||
status.success(),
|
||||
"perf stat run exited nonzero: {shell_cmd}"
|
||||
);
|
||||
let text = std::fs::read_to_string(out_file)
|
||||
.with_context(|| format!("perf stat output missing: {}", out_file.display()))?;
|
||||
Ok((elapsed, parse_perf_csv(&text)))
|
||||
}
|
||||
|
||||
/// Interleaved A/B with each timed run wrapped in `perf stat`. The perf
|
||||
/// wrapper overhead lands on both sides identically, but this path is
|
||||
/// diagnostics only — verdicts always measure through the unwrapped
|
||||
/// `exec::run_interleaved`.
|
||||
pub fn run_interleaved_perf(
|
||||
baseline_cmd: &str,
|
||||
candidate_cmd: &str,
|
||||
runs_per_side: usize,
|
||||
warmup: usize,
|
||||
) -> anyhow::Result<(MeasuredSamples, PerfReport)> {
|
||||
anyhow::ensure!(cfg!(unix), "--perf-stat requires a Unix host with perf(1)");
|
||||
let tmp = std::env::temp_dir().join(format!("bench-runner-perf-{}.csv", std::process::id()));
|
||||
let mut samples = MeasuredSamples {
|
||||
baseline_s: Vec::with_capacity(runs_per_side),
|
||||
candidate_s: Vec::with_capacity(runs_per_side),
|
||||
baseline_max_rss_kib: Vec::new(),
|
||||
candidate_max_rss_kib: Vec::new(),
|
||||
};
|
||||
let mut report = PerfReport::default();
|
||||
for (side, measured) in interleaved(runs_per_side, warmup) {
|
||||
let cmd = match side {
|
||||
Side::Baseline => baseline_cmd,
|
||||
Side::Candidate => candidate_cmd,
|
||||
};
|
||||
let (elapsed, parsed) = time_once_perf(cmd, &tmp)?;
|
||||
if measured {
|
||||
match side {
|
||||
Side::Baseline => {
|
||||
samples.baseline_s.push(elapsed);
|
||||
report.baseline.absorb(&parsed);
|
||||
}
|
||||
Side::Candidate => {
|
||||
samples.candidate_s.push(elapsed);
|
||||
report.candidate.absorb(&parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
Ok((samples, report))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const CANNED: &str = "\
|
||||
# started on Mon Jul 13 12:00:00 2026
|
||||
|
||||
123456789,,cycles,401000000,100.00,,
|
||||
234567890,,instructions:u,401000000,100.00,1.90,insn per cycle
|
||||
<not supported>,,cache-misses,0,100.00,,
|
||||
<not counted>,,branch-misses,0,0.00,,
|
||||
garbage line without commas
|
||||
999,,unrelated-event,1,100.00,,
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn parses_values_and_skips_junk() {
|
||||
let parsed = parse_perf_csv(CANNED);
|
||||
assert_eq!(
|
||||
parsed,
|
||||
vec![
|
||||
("cycles".to_string(), 123_456_789),
|
||||
("instructions:u".to_string(), 234_567_890),
|
||||
("unrelated-event".to_string(), 999),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_index_tolerates_modifier_suffix_only() {
|
||||
assert_eq!(event_index("cycles"), Some(0));
|
||||
assert_eq!(event_index("instructions:u"), Some(1));
|
||||
assert_eq!(event_index("cache-misses"), Some(2));
|
||||
assert_eq!(event_index("cycle"), None);
|
||||
assert_eq!(event_index("cycles_total"), None);
|
||||
assert_eq!(event_index("unrelated-event"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absorb_buckets_by_event() {
|
||||
let mut side = PerfSide::default();
|
||||
side.absorb(&parse_perf_csv(CANNED));
|
||||
assert_eq!(side.samples[0], vec![123_456_789]);
|
||||
assert_eq!(side.samples[1], vec![234_567_890]);
|
||||
assert!(side.samples[2].is_empty(), "<not supported> is skipped");
|
||||
assert!(side.samples[3].is_empty(), "<not counted> is skipped");
|
||||
}
|
||||
}
|
||||
+106
-11
@@ -1,29 +1,87 @@
|
||||
//! Command execution and timing. Whole-program mode (hyperfine-style):
|
||||
//! spawn the command, measure wall time, discard warm-ups per the
|
||||
//! schedule. `perf stat` counters and RAPL energy are Phase 1 follow-ups
|
||||
//! (tracked in README roadmap); wall time alone is enough to bring up
|
||||
//! A/A calibration.
|
||||
//! schedule. Max RSS is captured per run via `wait4` on Unix; opt-in
|
||||
//! `perf stat` counters live in `counters` (diagnostics only).
|
||||
|
||||
use crate::schedule::{interleaved, Side};
|
||||
use anyhow::Context;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::process::{Child, Command, ExitStatus, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MeasuredSamples {
|
||||
pub baseline_s: Vec<f64>,
|
||||
pub candidate_s: Vec<f64>,
|
||||
/// Per-run max resident set size, normalized to KiB from `wait4`
|
||||
/// ru_maxrss (whose own unit is platform-defined — see
|
||||
/// `maxrss_to_kib`). Empty on non-Unix hosts.
|
||||
pub baseline_max_rss_kib: Vec<u64>,
|
||||
pub candidate_max_rss_kib: Vec<u64>,
|
||||
}
|
||||
|
||||
fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<f64> {
|
||||
/// Raw `ru_maxrss` → KiB.
|
||||
///
|
||||
/// The unit of `ru_maxrss` is *platform-defined*, and the platforms
|
||||
/// disagree: Linux and the BSDs report kilobytes, while Apple platforms
|
||||
/// (macOS et al.) report **bytes**. Taking the raw value as KiB everywhere
|
||||
/// — as this did — silently inflates every macOS reading by 1024×, and the
|
||||
/// number is not cosmetic: verdict records its median as
|
||||
/// `env_fingerprint.max_rss_kib` in the ledger, where an allocator-swap
|
||||
/// attempt is judged partly on memory. Everything downstream is KiB, so the
|
||||
/// raw value is normalized here, at the syscall edge.
|
||||
///
|
||||
/// `raw_is_bytes` is the platform predicate (passed at the one call site, so
|
||||
/// this stays a pure function testable on every host). Bytes round *up*: a
|
||||
/// nonzero RSS must never record as 0 KiB. Negative values — a kernel that
|
||||
/// reports nothing — clamp to 0 rather than wrapping into the exabytes.
|
||||
#[cfg(any(unix, test))]
|
||||
fn maxrss_to_kib(raw: i64, raw_is_bytes: bool) -> u64 {
|
||||
let raw = raw.max(0) as u64;
|
||||
if raw_is_bytes {
|
||||
raw.div_ceil(1024)
|
||||
} else {
|
||||
raw
|
||||
}
|
||||
}
|
||||
|
||||
/// Reap the child, returning its exit status and max RSS in KiB when the
|
||||
/// platform reports it.
|
||||
#[cfg(unix)]
|
||||
fn wait_with_rusage(child: &mut Child) -> anyhow::Result<(ExitStatus, Option<u64>)> {
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
let pid = child.id() as libc::pid_t;
|
||||
let mut status: libc::c_int = 0;
|
||||
let mut usage: libc::rusage = unsafe { std::mem::zeroed() };
|
||||
loop {
|
||||
let r = unsafe { libc::wait4(pid, &mut status, 0, &mut usage) };
|
||||
if r == pid {
|
||||
break;
|
||||
}
|
||||
let err = std::io::Error::last_os_error();
|
||||
if err.raw_os_error() != Some(libc::EINTR) {
|
||||
return Err(err).context("wait4 failed");
|
||||
}
|
||||
}
|
||||
// Apple's getrusage reports ru_maxrss in bytes; Linux and the BSDs in KiB.
|
||||
let max_rss_kib = maxrss_to_kib(usage.ru_maxrss as i64, cfg!(target_vendor = "apple"));
|
||||
Ok((ExitStatus::from_raw(status), Some(max_rss_kib)))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn wait_with_rusage(child: &mut Child) -> anyhow::Result<(ExitStatus, Option<u64>)> {
|
||||
Ok((child.wait()?, None))
|
||||
}
|
||||
|
||||
fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<(f64, Option<u64>)> {
|
||||
let start = Instant::now();
|
||||
let status = Command::new("sh")
|
||||
let mut child = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(shell_cmd)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to spawn: {shell_cmd}"))?;
|
||||
let (status, max_rss_kib) = wait_with_rusage(&mut child)?;
|
||||
if inject_spin_s > 0.0 {
|
||||
// Calibration-only: a real busy-wait inside the timed window, so an
|
||||
// injected slowdown flows through the entire measurement pipeline
|
||||
@@ -35,7 +93,7 @@ fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<f64> {
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}");
|
||||
Ok(elapsed)
|
||||
Ok((elapsed, max_rss_kib))
|
||||
}
|
||||
|
||||
/// Run baseline and candidate commands on the interleaved schedule and
|
||||
@@ -53,19 +111,56 @@ pub fn run_interleaved(
|
||||
let mut out = MeasuredSamples {
|
||||
baseline_s: Vec::with_capacity(runs_per_side),
|
||||
candidate_s: Vec::with_capacity(runs_per_side),
|
||||
baseline_max_rss_kib: Vec::new(),
|
||||
candidate_max_rss_kib: Vec::new(),
|
||||
};
|
||||
for (side, measured) in interleaved(runs_per_side, warmup) {
|
||||
let (cmd, spin) = match side {
|
||||
Side::Baseline => (baseline_cmd, 0.0),
|
||||
Side::Candidate => (candidate_cmd, inject_candidate_spin_s),
|
||||
};
|
||||
let elapsed = time_once(cmd, spin)?;
|
||||
let (elapsed, max_rss_kib) = time_once(cmd, spin)?;
|
||||
if measured {
|
||||
match side {
|
||||
Side::Baseline => out.baseline_s.push(elapsed),
|
||||
Side::Candidate => out.candidate_s.push(elapsed),
|
||||
Side::Baseline => {
|
||||
out.baseline_s.push(elapsed);
|
||||
out.baseline_max_rss_kib.extend(max_rss_kib);
|
||||
}
|
||||
Side::Candidate => {
|
||||
out.candidate_s.push(elapsed);
|
||||
out.candidate_max_rss_kib.extend(max_rss_kib);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::maxrss_to_kib;
|
||||
|
||||
#[test]
|
||||
fn maxrss_kib_passthrough_when_platform_reports_kib() {
|
||||
// Linux / BSD: already KiB.
|
||||
assert_eq!(maxrss_to_kib(2048, false), 2048);
|
||||
assert_eq!(maxrss_to_kib(0, false), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maxrss_bytes_are_converted_not_recorded_raw() {
|
||||
// Apple: bytes. 2 MiB is 2048 KiB, not 2_097_152 — the 1024×
|
||||
// inflation this guards against.
|
||||
assert_eq!(maxrss_to_kib(2_097_152, true), 2048);
|
||||
// Rounds up: a nonzero RSS never records as 0 KiB.
|
||||
assert_eq!(maxrss_to_kib(1, true), 1);
|
||||
assert_eq!(maxrss_to_kib(1025, true), 2);
|
||||
assert_eq!(maxrss_to_kib(0, true), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maxrss_negative_clamps_instead_of_wrapping() {
|
||||
assert_eq!(maxrss_to_kib(-1, false), 0);
|
||||
assert_eq!(maxrss_to_kib(-1, true), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
//! The product is this crate: interleaved A/B measurement, bootstrap CIs
|
||||
//! over the ratio of medians, threshold-gated accept decisions, and A/A
|
||||
//! self-calibration. Pure statistics live here and are unit-tested;
|
||||
//! process execution and `perf stat` integration live in `exec`.
|
||||
//! process execution lives in `exec`; opt-in `perf stat` diagnostics
|
||||
//! live in `counters`.
|
||||
|
||||
pub mod config;
|
||||
pub mod counters;
|
||||
pub mod exec;
|
||||
pub mod fingerprint;
|
||||
pub mod schedule;
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
||||
use bench_runner::{
|
||||
config::AcceptConfig, counters, decide, exec, fingerprint::EnvFingerprint, stats,
|
||||
};
|
||||
use clap::{Parser, Subcommand};
|
||||
use ledger::Verdict;
|
||||
use std::path::PathBuf;
|
||||
@@ -25,6 +27,11 @@ enum Cmd {
|
||||
baseline: String,
|
||||
#[arg(long)]
|
||||
candidate: String,
|
||||
/// Wrap each timed run in `perf stat` and print per-side counter
|
||||
/// medians (Unix only; opt-in diagnostics, never part of the
|
||||
/// accept decision).
|
||||
#[arg(long, default_value_t = false)]
|
||||
perf_stat: bool,
|
||||
},
|
||||
/// A/A self-test: same command both sides; must yield a null verdict.
|
||||
Aa {
|
||||
@@ -429,12 +436,13 @@ fn main() -> Result<()> {
|
||||
None => AcceptConfig::default(),
|
||||
};
|
||||
|
||||
let (baseline, candidate, aa_mode) = match &cli.command {
|
||||
let (baseline, candidate, aa_mode, perf_stat) = match &cli.command {
|
||||
Cmd::Compare {
|
||||
baseline,
|
||||
candidate,
|
||||
} => (baseline.clone(), candidate.clone(), false),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
||||
perf_stat,
|
||||
} => (baseline.clone(), candidate.clone(), false, *perf_stat),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true, false),
|
||||
Cmd::Calibrate {
|
||||
cmd,
|
||||
sessions,
|
||||
@@ -502,13 +510,24 @@ fn main() -> Result<()> {
|
||||
cfg.runs_per_side, cfg.warmup_runs
|
||||
);
|
||||
|
||||
let samples = exec::run_interleaved(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
0.0,
|
||||
)?;
|
||||
let (samples, perf_report) = if perf_stat {
|
||||
let (samples, report) = counters::run_interleaved_perf(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
)?;
|
||||
(samples, Some(report))
|
||||
} else {
|
||||
let samples = exec::run_interleaved(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
0.0,
|
||||
)?;
|
||||
(samples, None)
|
||||
};
|
||||
let ci = stats::bootstrap_ratio_ci(
|
||||
&samples.baseline_s,
|
||||
&samples.candidate_s,
|
||||
@@ -524,6 +543,9 @@ fn main() -> Result<()> {
|
||||
ci.lo,
|
||||
ci.hi
|
||||
);
|
||||
if let Some(report) = &perf_report {
|
||||
report.print(&baseline, &candidate);
|
||||
}
|
||||
|
||||
if aa_mode {
|
||||
// Null verdict required: an "accept" here is a calibration failure.
|
||||
|
||||
@@ -147,6 +147,7 @@ fn spawn_server(cfg: &ServiceCfg, port: u16) -> Result<Child> {
|
||||
Ok(child)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn set_linger_zero(s: &TcpStream) {
|
||||
// SO_LINGER with a 0 timeout → RST on close instead of FIN, so the
|
||||
// tens of thousands of short client connections a load session opens
|
||||
@@ -170,6 +171,12 @@ fn set_linger_zero(s: &TcpStream) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn set_linger_zero(_s: &TcpStream) {
|
||||
// Service mode is POSIX-only at runtime (`sh`, taskset); this stub
|
||||
// only keeps the crate compiling on non-Unix hosts.
|
||||
}
|
||||
|
||||
fn one_request(port: u16) -> std::io::Result<()> {
|
||||
let mut s = TcpStream::connect((Ipv4Addr::LOCALHOST, port))?;
|
||||
s.set_nodelay(true)?;
|
||||
|
||||
Reference in New Issue
Block a user