mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
gates: close the audit findings — enforce the gates the docs claimed
An adversarial audit found the enforcement story ahead of the code: gates that were documented as hard gates were advisory, manual, or absent, and several human-facing claims outran the machine record. This makes the claims true. High findings - Differential fuzz was documentation, not enforcement: verdict hardcoded fuzz_iters=0 and diff-test hardcoded the gate to Skipped, while the case studies claimed 10k-iteration fuzz on the same wins. Fuzz is now a real pipeline gate (strict FUZZ-RESULT contract, executed count recorded), and it is inherently baseline-vs-candidate, so it runs on the accept path and reports Skipped in the standalone `just gates` flow. New hard rule: a machine `accepted` verdict is impossible unless the fuzz gate actually passed — otherwise it caps at needs-human-review. - The agent's system prompt claimed FP/concurrency/unsafe patches were "routed to human review automatically". No such mechanism existed. Added a conservative lexical risk classifier (crates/verdict/src/risk.rs) that forces needs-human-review on any would-be accept carrying those signals; fp-tolerance equivalence always signals. - The local verdict path ran agent-influenced code unsandboxed. It now runs under scripts/no-net.sh (network namespace) and fails closed. Upstream test suites are hash-verified against corpora/<t>/TESTSUITE.sha256 when present (scripts/pin-testsuite.sh generates them). Also fixed, found while fixing the above - The pipeline script's `set -e` was inert (brace group as the left operand of `&&`), so a failing candidate build did not abort the run and verdict could bench a stale binary. Now a scoped subshell whose status is branched on at top level. - A wrapper that exits before exec'ing (no-net.sh refusing, bad wrapper path) stranded RUNNING.lock and left read_verdict polling forever. Failure accounting now lives outside the wrapper. - harnessd's patch allowlist ignored git rename/copy headers and --no-prefix diffs; the rename-escape case is now a committed regression test. - The workspace did not compile on Windows (ungated Unix APIs). cfg-gated; 71 portable tests now run on both platforms. - pin-check regenerated the corpus manifest instead of verifying it; it is now verify-only, with a deliberate `just pin-corpus` to re-pin. - Max RSS is captured per run on Unix; PMU counters are opt-in diagnostics (compare --perf-stat), never part of the accept decision. Docs now match the code, including what is NOT yet enforced: no target ships a TESTSUITE.sha256 pin, no target declares [build].tsan, and matmul is not wired to the fp-tolerance gate — each is stated as mechanized-but-inert rather than done. Corrected the stale "33 ledger rows" (34), the playbook's class-3 cJSON win (rejected), adversarial-review.md's sanitizer-separation claim (five accepted rows carry sanitizers_clean=false), the unsupported "40,000 requests" figure, and the blanket "historical rows record fuzz_iters=0" (8 of 10; phase0-tokei-002/-003 record 4,332). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -10,7 +10,9 @@ repository.workspace = true
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
ledger = { path = "../ledger" }
|
||||
libc = "0.2"
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
@@ -23,6 +23,13 @@ pub struct AcceptConfig {
|
||||
/// both sides identically.
|
||||
#[serde(default)]
|
||||
pub pin_prefix: String,
|
||||
/// Differential-fuzz iterations requested per gate run (SPEC §3.2).
|
||||
#[serde(default = "default_fuzz_iters")]
|
||||
pub fuzz_iters: u64,
|
||||
}
|
||||
|
||||
fn default_fuzz_iters() -> u64 {
|
||||
10_000
|
||||
}
|
||||
|
||||
impl Default for AcceptConfig {
|
||||
@@ -35,6 +42,7 @@ impl Default for AcceptConfig {
|
||||
confidence: 0.95,
|
||||
bootstrap_seed: 0x707E_17E5,
|
||||
pin_prefix: String::new(),
|
||||
fuzz_iters: default_fuzz_iters(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,3 +60,31 @@ impl AcceptConfig {
|
||||
Ok(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const MINIMAL: &str = "threshold = 0.02\nruns_per_side = 30\nwarmup_runs = 3\n\
|
||||
bootstrap_iters = 10000\nconfidence = 0.95\nbootstrap_seed = 7\n";
|
||||
|
||||
#[test]
|
||||
fn fuzz_iters_defaults_when_absent() {
|
||||
let cfg: AcceptConfig = toml::from_str(MINIMAL).unwrap();
|
||||
assert_eq!(cfg.fuzz_iters, 10_000);
|
||||
assert_eq!(AcceptConfig::default().fuzz_iters, 10_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_iters_overridable() {
|
||||
let raw = format!("{MINIMAL}fuzz_iters = 500\n");
|
||||
let cfg: AcceptConfig = toml::from_str(&raw).unwrap();
|
||||
assert_eq!(cfg.fuzz_iters, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_still_rejected() {
|
||||
let raw = format!("{MINIMAL}fuz_iters = 500\n");
|
||||
assert!(toml::from_str::<AcceptConfig>(&raw).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Opt-in `perf stat` PMU counter capture. Diagnostics only — counters
|
||||
//! never feed the accept decision (wall time does, SPEC §3.1). Each
|
||||
//! timed run is wrapped `perf stat -x, -e <events> -o <tmpfile> -- sh -c
|
||||
//! <cmd>`; the CSV file is parsed after every run and per-side medians
|
||||
//! are printed with the standard caveats. Unix-only at runtime; the
|
||||
//! parser is portable and unit-tested.
|
||||
|
||||
use crate::exec::MeasuredSamples;
|
||||
use crate::schedule::{interleaved, Side};
|
||||
use crate::stats;
|
||||
use anyhow::Context;
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
pub const PERF_EVENTS: [&str; 4] = ["cycles", "instructions", "cache-misses", "branch-misses"];
|
||||
|
||||
/// One side's counter samples: `samples[i]` holds one value per measured
|
||||
/// run for `PERF_EVENTS[i]`; runs where perf reported `<not counted>` or
|
||||
/// `<not supported>` contribute nothing.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PerfSide {
|
||||
pub samples: [Vec<u64>; 4],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PerfReport {
|
||||
pub baseline: PerfSide,
|
||||
pub candidate: PerfSide,
|
||||
}
|
||||
|
||||
/// Parse `perf stat -x,` CSV: field 0 is the counter value, field 2 the
|
||||
/// event name (perf may append a modifier, e.g. `cycles:u`). Comment
|
||||
/// lines, blank lines, and non-numeric values (`<not counted>`,
|
||||
/// `<not supported>`) are skipped. Returns (event, count) in file order.
|
||||
pub fn parse_perf_csv(text: &str) -> Vec<(String, u64)> {
|
||||
let mut out = Vec::new();
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let mut fields = line.split(',');
|
||||
let value = fields.next().unwrap_or("");
|
||||
let _unit = fields.next();
|
||||
let Some(event) = fields.next() else {
|
||||
continue;
|
||||
};
|
||||
let Ok(v) = value.trim().parse::<u64>() else {
|
||||
continue;
|
||||
};
|
||||
out.push((event.trim().to_string(), v));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Index into PERF_EVENTS, tolerating a perf modifier suffix
|
||||
/// (`cycles:u` when perf_event_paranoid restricts kernel counting).
|
||||
fn event_index(event: &str) -> Option<usize> {
|
||||
PERF_EVENTS.iter().position(|name| {
|
||||
event == *name || (event.starts_with(name) && event[name.len()..].starts_with(':'))
|
||||
})
|
||||
}
|
||||
|
||||
impl PerfSide {
|
||||
fn absorb(&mut self, parsed: &[(String, u64)]) {
|
||||
for (event, value) in parsed {
|
||||
if let Some(i) = event_index(event) {
|
||||
self.samples[i].push(*value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PerfReport {
|
||||
/// Per-side medians, each with its command (workload) and run count.
|
||||
pub fn print(&self, baseline_cmd: &str, candidate_cmd: &str) {
|
||||
println!("perf-stat per-side medians:");
|
||||
for (label, cmd, side) in [
|
||||
("baseline ", baseline_cmd, &self.baseline),
|
||||
("candidate", candidate_cmd, &self.candidate),
|
||||
] {
|
||||
let cols: Vec<String> = PERF_EVENTS
|
||||
.iter()
|
||||
.zip(&side.samples)
|
||||
.map(|(name, xs)| {
|
||||
if xs.is_empty() {
|
||||
format!("{name}=n/a")
|
||||
} else {
|
||||
let v: Vec<f64> = xs.iter().map(|&x| x as f64).collect();
|
||||
format!(
|
||||
"{name}={:.0} (median of {} runs)",
|
||||
stats::median(&v),
|
||||
xs.len()
|
||||
)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
println!(" {label} `{cmd}`: {}", cols.join(", "));
|
||||
}
|
||||
println!(
|
||||
" note: PMU counters unavailable in most VMs — flag is opt-in diagnostics, \
|
||||
not part of the accept decision"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn time_once_perf(shell_cmd: &str, out_file: &Path) -> anyhow::Result<(f64, Vec<(String, u64)>)> {
|
||||
let start = Instant::now();
|
||||
let status = Command::new("perf")
|
||||
.args(["stat", "-x,", "-e", &PERF_EVENTS.join(",")])
|
||||
.arg("-o")
|
||||
.arg(out_file)
|
||||
.args(["--", "sh", "-c", shell_cmd])
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.with_context(|| format!("failed to spawn perf stat for: {shell_cmd}"))?;
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
anyhow::ensure!(
|
||||
status.success(),
|
||||
"perf stat run exited nonzero: {shell_cmd}"
|
||||
);
|
||||
let text = std::fs::read_to_string(out_file)
|
||||
.with_context(|| format!("perf stat output missing: {}", out_file.display()))?;
|
||||
Ok((elapsed, parse_perf_csv(&text)))
|
||||
}
|
||||
|
||||
/// Interleaved A/B with each timed run wrapped in `perf stat`. The perf
|
||||
/// wrapper overhead lands on both sides identically, but this path is
|
||||
/// diagnostics only — verdicts always measure through the unwrapped
|
||||
/// `exec::run_interleaved`.
|
||||
pub fn run_interleaved_perf(
|
||||
baseline_cmd: &str,
|
||||
candidate_cmd: &str,
|
||||
runs_per_side: usize,
|
||||
warmup: usize,
|
||||
) -> anyhow::Result<(MeasuredSamples, PerfReport)> {
|
||||
anyhow::ensure!(cfg!(unix), "--perf-stat requires a Unix host with perf(1)");
|
||||
let tmp = std::env::temp_dir().join(format!("bench-runner-perf-{}.csv", std::process::id()));
|
||||
let mut samples = MeasuredSamples {
|
||||
baseline_s: Vec::with_capacity(runs_per_side),
|
||||
candidate_s: Vec::with_capacity(runs_per_side),
|
||||
baseline_max_rss_kib: Vec::new(),
|
||||
candidate_max_rss_kib: Vec::new(),
|
||||
};
|
||||
let mut report = PerfReport::default();
|
||||
for (side, measured) in interleaved(runs_per_side, warmup) {
|
||||
let cmd = match side {
|
||||
Side::Baseline => baseline_cmd,
|
||||
Side::Candidate => candidate_cmd,
|
||||
};
|
||||
let (elapsed, parsed) = time_once_perf(cmd, &tmp)?;
|
||||
if measured {
|
||||
match side {
|
||||
Side::Baseline => {
|
||||
samples.baseline_s.push(elapsed);
|
||||
report.baseline.absorb(&parsed);
|
||||
}
|
||||
Side::Candidate => {
|
||||
samples.candidate_s.push(elapsed);
|
||||
report.candidate.absorb(&parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
Ok((samples, report))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const CANNED: &str = "\
|
||||
# started on Mon Jul 13 12:00:00 2026
|
||||
|
||||
123456789,,cycles,401000000,100.00,,
|
||||
234567890,,instructions:u,401000000,100.00,1.90,insn per cycle
|
||||
<not supported>,,cache-misses,0,100.00,,
|
||||
<not counted>,,branch-misses,0,0.00,,
|
||||
garbage line without commas
|
||||
999,,unrelated-event,1,100.00,,
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn parses_values_and_skips_junk() {
|
||||
let parsed = parse_perf_csv(CANNED);
|
||||
assert_eq!(
|
||||
parsed,
|
||||
vec![
|
||||
("cycles".to_string(), 123_456_789),
|
||||
("instructions:u".to_string(), 234_567_890),
|
||||
("unrelated-event".to_string(), 999),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_index_tolerates_modifier_suffix_only() {
|
||||
assert_eq!(event_index("cycles"), Some(0));
|
||||
assert_eq!(event_index("instructions:u"), Some(1));
|
||||
assert_eq!(event_index("cache-misses"), Some(2));
|
||||
assert_eq!(event_index("cycle"), None);
|
||||
assert_eq!(event_index("cycles_total"), None);
|
||||
assert_eq!(event_index("unrelated-event"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absorb_buckets_by_event() {
|
||||
let mut side = PerfSide::default();
|
||||
side.absorb(&parse_perf_csv(CANNED));
|
||||
assert_eq!(side.samples[0], vec![123_456_789]);
|
||||
assert_eq!(side.samples[1], vec![234_567_890]);
|
||||
assert!(side.samples[2].is_empty(), "<not supported> is skipped");
|
||||
assert!(side.samples[3].is_empty(), "<not counted> is skipped");
|
||||
}
|
||||
}
|
||||
+106
-11
@@ -1,29 +1,87 @@
|
||||
//! Command execution and timing. Whole-program mode (hyperfine-style):
|
||||
//! spawn the command, measure wall time, discard warm-ups per the
|
||||
//! schedule. `perf stat` counters and RAPL energy are Phase 1 follow-ups
|
||||
//! (tracked in README roadmap); wall time alone is enough to bring up
|
||||
//! A/A calibration.
|
||||
//! schedule. Max RSS is captured per run via `wait4` on Unix; opt-in
|
||||
//! `perf stat` counters live in `counters` (diagnostics only).
|
||||
|
||||
use crate::schedule::{interleaved, Side};
|
||||
use anyhow::Context;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::process::{Child, Command, ExitStatus, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MeasuredSamples {
|
||||
pub baseline_s: Vec<f64>,
|
||||
pub candidate_s: Vec<f64>,
|
||||
/// Per-run max resident set size, normalized to KiB from `wait4`
|
||||
/// ru_maxrss (whose own unit is platform-defined — see
|
||||
/// `maxrss_to_kib`). Empty on non-Unix hosts.
|
||||
pub baseline_max_rss_kib: Vec<u64>,
|
||||
pub candidate_max_rss_kib: Vec<u64>,
|
||||
}
|
||||
|
||||
fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<f64> {
|
||||
/// Raw `ru_maxrss` → KiB.
|
||||
///
|
||||
/// The unit of `ru_maxrss` is *platform-defined*, and the platforms
|
||||
/// disagree: Linux and the BSDs report kilobytes, while Apple platforms
|
||||
/// (macOS et al.) report **bytes**. Taking the raw value as KiB everywhere
|
||||
/// — as this did — silently inflates every macOS reading by 1024×, and the
|
||||
/// number is not cosmetic: verdict records its median as
|
||||
/// `env_fingerprint.max_rss_kib` in the ledger, where an allocator-swap
|
||||
/// attempt is judged partly on memory. Everything downstream is KiB, so the
|
||||
/// raw value is normalized here, at the syscall edge.
|
||||
///
|
||||
/// `raw_is_bytes` is the platform predicate (passed at the one call site, so
|
||||
/// this stays a pure function testable on every host). Bytes round *up*: a
|
||||
/// nonzero RSS must never record as 0 KiB. Negative values — a kernel that
|
||||
/// reports nothing — clamp to 0 rather than wrapping into the exabytes.
|
||||
#[cfg(any(unix, test))]
|
||||
fn maxrss_to_kib(raw: i64, raw_is_bytes: bool) -> u64 {
|
||||
let raw = raw.max(0) as u64;
|
||||
if raw_is_bytes {
|
||||
raw.div_ceil(1024)
|
||||
} else {
|
||||
raw
|
||||
}
|
||||
}
|
||||
|
||||
/// Reap the child, returning its exit status and max RSS in KiB when the
|
||||
/// platform reports it.
|
||||
#[cfg(unix)]
|
||||
fn wait_with_rusage(child: &mut Child) -> anyhow::Result<(ExitStatus, Option<u64>)> {
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
let pid = child.id() as libc::pid_t;
|
||||
let mut status: libc::c_int = 0;
|
||||
let mut usage: libc::rusage = unsafe { std::mem::zeroed() };
|
||||
loop {
|
||||
let r = unsafe { libc::wait4(pid, &mut status, 0, &mut usage) };
|
||||
if r == pid {
|
||||
break;
|
||||
}
|
||||
let err = std::io::Error::last_os_error();
|
||||
if err.raw_os_error() != Some(libc::EINTR) {
|
||||
return Err(err).context("wait4 failed");
|
||||
}
|
||||
}
|
||||
// Apple's getrusage reports ru_maxrss in bytes; Linux and the BSDs in KiB.
|
||||
let max_rss_kib = maxrss_to_kib(usage.ru_maxrss as i64, cfg!(target_vendor = "apple"));
|
||||
Ok((ExitStatus::from_raw(status), Some(max_rss_kib)))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn wait_with_rusage(child: &mut Child) -> anyhow::Result<(ExitStatus, Option<u64>)> {
|
||||
Ok((child.wait()?, None))
|
||||
}
|
||||
|
||||
fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<(f64, Option<u64>)> {
|
||||
let start = Instant::now();
|
||||
let status = Command::new("sh")
|
||||
let mut child = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(shell_cmd)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to spawn: {shell_cmd}"))?;
|
||||
let (status, max_rss_kib) = wait_with_rusage(&mut child)?;
|
||||
if inject_spin_s > 0.0 {
|
||||
// Calibration-only: a real busy-wait inside the timed window, so an
|
||||
// injected slowdown flows through the entire measurement pipeline
|
||||
@@ -35,7 +93,7 @@ fn time_once(shell_cmd: &str, inject_spin_s: f64) -> anyhow::Result<f64> {
|
||||
}
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}");
|
||||
Ok(elapsed)
|
||||
Ok((elapsed, max_rss_kib))
|
||||
}
|
||||
|
||||
/// Run baseline and candidate commands on the interleaved schedule and
|
||||
@@ -53,19 +111,56 @@ pub fn run_interleaved(
|
||||
let mut out = MeasuredSamples {
|
||||
baseline_s: Vec::with_capacity(runs_per_side),
|
||||
candidate_s: Vec::with_capacity(runs_per_side),
|
||||
baseline_max_rss_kib: Vec::new(),
|
||||
candidate_max_rss_kib: Vec::new(),
|
||||
};
|
||||
for (side, measured) in interleaved(runs_per_side, warmup) {
|
||||
let (cmd, spin) = match side {
|
||||
Side::Baseline => (baseline_cmd, 0.0),
|
||||
Side::Candidate => (candidate_cmd, inject_candidate_spin_s),
|
||||
};
|
||||
let elapsed = time_once(cmd, spin)?;
|
||||
let (elapsed, max_rss_kib) = time_once(cmd, spin)?;
|
||||
if measured {
|
||||
match side {
|
||||
Side::Baseline => out.baseline_s.push(elapsed),
|
||||
Side::Candidate => out.candidate_s.push(elapsed),
|
||||
Side::Baseline => {
|
||||
out.baseline_s.push(elapsed);
|
||||
out.baseline_max_rss_kib.extend(max_rss_kib);
|
||||
}
|
||||
Side::Candidate => {
|
||||
out.candidate_s.push(elapsed);
|
||||
out.candidate_max_rss_kib.extend(max_rss_kib);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::maxrss_to_kib;
|
||||
|
||||
#[test]
|
||||
fn maxrss_kib_passthrough_when_platform_reports_kib() {
|
||||
// Linux / BSD: already KiB.
|
||||
assert_eq!(maxrss_to_kib(2048, false), 2048);
|
||||
assert_eq!(maxrss_to_kib(0, false), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maxrss_bytes_are_converted_not_recorded_raw() {
|
||||
// Apple: bytes. 2 MiB is 2048 KiB, not 2_097_152 — the 1024×
|
||||
// inflation this guards against.
|
||||
assert_eq!(maxrss_to_kib(2_097_152, true), 2048);
|
||||
// Rounds up: a nonzero RSS never records as 0 KiB.
|
||||
assert_eq!(maxrss_to_kib(1, true), 1);
|
||||
assert_eq!(maxrss_to_kib(1025, true), 2);
|
||||
assert_eq!(maxrss_to_kib(0, true), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maxrss_negative_clamps_instead_of_wrapping() {
|
||||
assert_eq!(maxrss_to_kib(-1, false), 0);
|
||||
assert_eq!(maxrss_to_kib(-1, true), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
//! The product is this crate: interleaved A/B measurement, bootstrap CIs
|
||||
//! over the ratio of medians, threshold-gated accept decisions, and A/A
|
||||
//! self-calibration. Pure statistics live here and are unit-tested;
|
||||
//! process execution and `perf stat` integration live in `exec`.
|
||||
//! process execution lives in `exec`; opt-in `perf stat` diagnostics
|
||||
//! live in `counters`.
|
||||
|
||||
pub mod config;
|
||||
pub mod counters;
|
||||
pub mod exec;
|
||||
pub mod fingerprint;
|
||||
pub mod schedule;
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
||||
use bench_runner::{
|
||||
config::AcceptConfig, counters, decide, exec, fingerprint::EnvFingerprint, stats,
|
||||
};
|
||||
use clap::{Parser, Subcommand};
|
||||
use ledger::Verdict;
|
||||
use std::path::PathBuf;
|
||||
@@ -25,6 +27,11 @@ enum Cmd {
|
||||
baseline: String,
|
||||
#[arg(long)]
|
||||
candidate: String,
|
||||
/// Wrap each timed run in `perf stat` and print per-side counter
|
||||
/// medians (Unix only; opt-in diagnostics, never part of the
|
||||
/// accept decision).
|
||||
#[arg(long, default_value_t = false)]
|
||||
perf_stat: bool,
|
||||
},
|
||||
/// A/A self-test: same command both sides; must yield a null verdict.
|
||||
Aa {
|
||||
@@ -429,12 +436,13 @@ fn main() -> Result<()> {
|
||||
None => AcceptConfig::default(),
|
||||
};
|
||||
|
||||
let (baseline, candidate, aa_mode) = match &cli.command {
|
||||
let (baseline, candidate, aa_mode, perf_stat) = match &cli.command {
|
||||
Cmd::Compare {
|
||||
baseline,
|
||||
candidate,
|
||||
} => (baseline.clone(), candidate.clone(), false),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
||||
perf_stat,
|
||||
} => (baseline.clone(), candidate.clone(), false, *perf_stat),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true, false),
|
||||
Cmd::Calibrate {
|
||||
cmd,
|
||||
sessions,
|
||||
@@ -502,13 +510,24 @@ fn main() -> Result<()> {
|
||||
cfg.runs_per_side, cfg.warmup_runs
|
||||
);
|
||||
|
||||
let samples = exec::run_interleaved(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
0.0,
|
||||
)?;
|
||||
let (samples, perf_report) = if perf_stat {
|
||||
let (samples, report) = counters::run_interleaved_perf(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
)?;
|
||||
(samples, Some(report))
|
||||
} else {
|
||||
let samples = exec::run_interleaved(
|
||||
&baseline,
|
||||
&candidate,
|
||||
cfg.runs_per_side,
|
||||
cfg.warmup_runs,
|
||||
0.0,
|
||||
)?;
|
||||
(samples, None)
|
||||
};
|
||||
let ci = stats::bootstrap_ratio_ci(
|
||||
&samples.baseline_s,
|
||||
&samples.candidate_s,
|
||||
@@ -524,6 +543,9 @@ fn main() -> Result<()> {
|
||||
ci.lo,
|
||||
ci.hi
|
||||
);
|
||||
if let Some(report) = &perf_report {
|
||||
report.print(&baseline, &candidate);
|
||||
}
|
||||
|
||||
if aa_mode {
|
||||
// Null verdict required: an "accept" here is a calibration failure.
|
||||
|
||||
@@ -147,6 +147,7 @@ fn spawn_server(cfg: &ServiceCfg, port: u16) -> Result<Child> {
|
||||
Ok(child)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn set_linger_zero(s: &TcpStream) {
|
||||
// SO_LINGER with a 0 timeout → RST on close instead of FIN, so the
|
||||
// tens of thousands of short client connections a load session opens
|
||||
@@ -170,6 +171,12 @@ fn set_linger_zero(s: &TcpStream) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn set_linger_zero(_s: &TcpStream) {
|
||||
// Service mode is POSIX-only at runtime (`sh`, taskset); this stub
|
||||
// only keeps the crate compiling on non-Unix hosts.
|
||||
}
|
||||
|
||||
fn one_request(port: u16) -> std::io::Result<()> {
|
||||
let mut s = TcpStream::connect((Ipv4Addr::LOCALHOST, port))?;
|
||||
s.set_nodelay(true)?;
|
||||
|
||||
@@ -43,6 +43,49 @@ pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool {
|
||||
.any(|(_, o)| matches!(o, GateOutcome::Passed))
|
||||
}
|
||||
|
||||
/// Inputs to one gate run: the candidate binary under test, the pristine
|
||||
/// baseline to differ it against, and the differential-fuzz iteration
|
||||
/// budget (from AcceptConfig, default 10_000).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct GateInputs<'a> {
|
||||
/// Patched binary (repo-root-relative): golden replay runs it, and it
|
||||
/// is the `{candidate}` side of differential fuzz.
|
||||
pub candidate_binary: &'a str,
|
||||
/// Pristine baseline binary (repo-root-relative), the `{baseline}` side
|
||||
/// of differential fuzz. `None` when the caller has no baseline to
|
||||
/// differ against — the standalone `just gates` flow builds only the
|
||||
/// working tree — and the fuzz gate is then skipped with that reason
|
||||
/// rather than silently comparing the candidate to itself. The accept
|
||||
/// path (`just verdict`) rebuilds a pristine baseline and passes it,
|
||||
/// and only a fuzz gate that actually Passed can mint an accept.
|
||||
pub baseline_binary: Option<&'a str>,
|
||||
pub fuzz_iters: u64,
|
||||
}
|
||||
|
||||
/// Everything one gate run produced: the four layers in fixed order
|
||||
/// (UpstreamTests, GoldenReplay, DifferentialFuzz, Sanitizers) plus the
|
||||
/// fuzz counts and equivalence mode that verdict records in the ledger.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct GateReport {
|
||||
pub gates: Vec<(GateLayer, GateOutcome)>,
|
||||
/// Fuzz iterations actually executed (parsed from the FUZZ-RESULT
|
||||
/// line); 0 when the fuzz gate was skipped or never reported.
|
||||
pub fuzz_iters: u64,
|
||||
pub fuzz_divergence: bool,
|
||||
/// "byte-identical" | "fp-tolerance"
|
||||
pub equivalence_mode: String,
|
||||
}
|
||||
|
||||
impl GateReport {
|
||||
pub fn outcome(&self, layer: GateLayer) -> Option<&GateOutcome> {
|
||||
self.gates.iter().find(|(l, _)| *l == layer).map(|(_, o)| o)
|
||||
}
|
||||
|
||||
pub fn all_passed(&self) -> bool {
|
||||
all_passed(&self.gates)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -71,4 +114,32 @@ mod tests {
|
||||
)];
|
||||
assert!(!all_passed(&results));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_outcome_looks_up_by_layer() {
|
||||
let report = GateReport {
|
||||
gates: vec![
|
||||
(GateLayer::UpstreamTests, GateOutcome::Passed),
|
||||
(
|
||||
GateLayer::GoldenReplay,
|
||||
GateOutcome::Failed {
|
||||
detail: "stdout diverged".into(),
|
||||
},
|
||||
),
|
||||
],
|
||||
fuzz_iters: 0,
|
||||
fuzz_divergence: false,
|
||||
equivalence_mode: "byte-identical".into(),
|
||||
};
|
||||
assert_eq!(
|
||||
report.outcome(GateLayer::UpstreamTests),
|
||||
Some(&GateOutcome::Passed)
|
||||
);
|
||||
assert!(matches!(
|
||||
report.outcome(GateLayer::GoldenReplay),
|
||||
Some(GateOutcome::Failed { .. })
|
||||
));
|
||||
assert_eq!(report.outcome(GateLayer::Sanitizers), None);
|
||||
assert!(!report.all_passed());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
//! Gate orchestration CLI: corpus pin → upstream tests → golden replay.
|
||||
//! Differential fuzz and sanitizers remain per-attempt manual steps in
|
||||
//! Phase 1 and are reported as skipped with a reason.
|
||||
//! Gate orchestration CLI: corpus + test-suite pins → upstream tests →
|
||||
//! policy-aware golden replay → differential fuzz. Sanitizers run on the
|
||||
//! accept path in verdict and are reported as skipped with a reason.
|
||||
//!
|
||||
//! Differential fuzz is skipped here too, and for the same structural
|
||||
//! reason: it differs a *pristine baseline* against the candidate, and this
|
||||
//! flow builds only the working tree — there is nothing to differ against.
|
||||
//! The accept path (`just verdict`) rebuilds a pristine baseline, passes it
|
||||
//! in, and refuses to mint an accept unless the fuzz gate actually passed.
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use diff_test::orchestrate::run_core_gates;
|
||||
use diff_test::target::{subst_out, TargetSpec};
|
||||
use diff_test::{all_passed, GateOutcome};
|
||||
use diff_test::{GateInputs, GateLayer, GateOutcome};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
@@ -21,6 +27,13 @@ struct Cli {
|
||||
/// tree is built to targets/<name>/gates-build and that is tested.
|
||||
#[arg(long)]
|
||||
binary: Option<String>,
|
||||
/// Differential-fuzz iteration budget, substituted for {iters} in the
|
||||
/// target's [gates].fuzz command. Inert on this flow: with no pristine
|
||||
/// baseline to differ against, the fuzz gate is skipped. The accept path
|
||||
/// (`just verdict`) rebuilds one and takes its budget from
|
||||
/// config/accept.toml.
|
||||
#[arg(long, default_value_t = 10_000)]
|
||||
fuzz_iters: u64,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
@@ -47,15 +60,42 @@ fn main() -> Result<()> {
|
||||
subst_out(&spec.build.binary, &out)
|
||||
}
|
||||
};
|
||||
let results = run_core_gates(&root, &spec, &binary)?;
|
||||
for (layer, outcome) in &results {
|
||||
let report = run_core_gates(
|
||||
&root,
|
||||
&spec,
|
||||
&GateInputs {
|
||||
candidate_binary: &binary,
|
||||
// No pristine baseline on this flow — see the module doc. The
|
||||
// fuzz gate reports Skipped with that reason rather than
|
||||
// comparing the candidate against itself.
|
||||
baseline_binary: None,
|
||||
fuzz_iters: cli.fuzz_iters,
|
||||
},
|
||||
)?;
|
||||
for (layer, outcome) in &report.gates {
|
||||
match outcome {
|
||||
GateOutcome::Passed if *layer == GateLayer::DifferentialFuzz => {
|
||||
println!("{layer:?}: PASS ({} iters)", report.fuzz_iters)
|
||||
}
|
||||
GateOutcome::Passed => println!("{layer:?}: PASS"),
|
||||
GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"),
|
||||
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
|
||||
}
|
||||
}
|
||||
if all_passed(&results) {
|
||||
if report.all_passed() {
|
||||
// Inside all_passed(), the fuzz gate is Passed or Skipped — never
|
||||
// Failed. Say so plainly: a PASS here must not read as a verified
|
||||
// differential-fuzz run, which is exactly the confusion that let
|
||||
// accepted rows carry fuzz_iters=0.
|
||||
if !matches!(
|
||||
report.outcome(GateLayer::DifferentialFuzz),
|
||||
Some(GateOutcome::Passed)
|
||||
) {
|
||||
println!(
|
||||
"note: differential fuzz did not run — `just gates` is not an accept path; only \
|
||||
`just verdict` (pristine baseline + fuzz gate) can accept a change"
|
||||
);
|
||||
}
|
||||
println!("gates: PASS ({})", cli.target);
|
||||
Ok(())
|
||||
} else {
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
//! Core gate sequence, callable from the diff-test CLI and from verdict:
|
||||
//! corpus pin (refuses on mismatch) → upstream tests → golden replay.
|
||||
//! corpus + test-suite pins (refuse on mismatch) → upstream tests →
|
||||
//! policy-aware golden replay → differential fuzz. Sanitizers run on the
|
||||
//! accept path in verdict and are reported here as skipped with a reason.
|
||||
//!
|
||||
//! Differential fuzz is the one gate that is not purely a property of the
|
||||
//! candidate: it differs the candidate against a *pristine baseline*, so it
|
||||
//! needs both binaries. Callers that have only a candidate (the standalone
|
||||
//! `just gates` flow) pass `baseline_binary: None` and the gate is skipped
|
||||
//! with that reason recorded — it is never faked by comparing the candidate
|
||||
//! against itself, and verdict refuses to mint an accept without it.
|
||||
|
||||
use crate::policy::EquivalencePolicy;
|
||||
use crate::target::{expected_golden_hash, TargetSpec};
|
||||
use crate::{pin, GateLayer, GateOutcome};
|
||||
use crate::{pin, GateInputs, GateLayer, GateOutcome, GateReport};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
fn sh(root: &Path, cmd: &str) -> anyhow::Result<std::process::Output> {
|
||||
@@ -17,14 +27,155 @@ fn sh(root: &Path, cmd: &str) -> anyhow::Result<std::process::Output> {
|
||||
.output()?)
|
||||
}
|
||||
|
||||
/// Run the mechanical gates for `target`, testing `binary` (repo-root-
|
||||
/// relative) in the golden replay. Corpus-pin mismatch is an error, not
|
||||
/// a gate failure: nothing runs on a tampered corpus.
|
||||
/// TESTSUITE.sha256 lives beside the corpus MANIFEST.sha256 (both under
|
||||
/// corpora/<target>/), so its path derives from the manifest's.
|
||||
fn suite_manifest_path(corpus_manifest: &Path) -> PathBuf {
|
||||
corpus_manifest
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new(""))
|
||||
.join("TESTSUITE.sha256")
|
||||
}
|
||||
|
||||
fn parse_u64_strict(s: &str) -> Option<u64> {
|
||||
if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
s.parse().ok()
|
||||
}
|
||||
|
||||
/// Parse the last `FUZZ-RESULT iters=<u64> divergences=<u64>` stdout line
|
||||
/// of a fuzz command (exact ASCII form, single spaces). Returns
|
||||
/// `(iters, divergences)`, or None when no line matches.
|
||||
pub fn parse_fuzz_result(stdout: &str) -> Option<(u64, u64)> {
|
||||
stdout.lines().rev().find_map(|line| {
|
||||
let line = line.strip_suffix('\r').unwrap_or(line);
|
||||
let rest = line.strip_prefix("FUZZ-RESULT iters=")?;
|
||||
let (iters, rest) = rest.split_once(' ')?;
|
||||
let divergences = rest.strip_prefix("divergences=")?;
|
||||
Some((parse_u64_strict(iters)?, parse_u64_strict(divergences)?))
|
||||
})
|
||||
}
|
||||
|
||||
/// Recorded verbatim when the target declares no fuzz command at all.
|
||||
const FUZZ_SKIP_NO_COMMAND: &str = "no fuzz command declared in target.toml [gates].fuzz";
|
||||
|
||||
/// Recorded verbatim when a fuzz command exists but the caller has no
|
||||
/// pristine baseline: differential fuzz differs two binaries, and comparing
|
||||
/// the candidate against itself would be a gate that cannot fail.
|
||||
const FUZZ_SKIP_NO_BASELINE: &str =
|
||||
"differential fuzz needs a pristine baseline binary; run the accept path (`just verdict`), \
|
||||
which rebuilds one";
|
||||
|
||||
/// Substitute the three `[gates].fuzz` template placeholders. The contract
|
||||
/// with `targets/<t>/target.toml` and `scripts/diff-fuzz-*.py`: `{iters}`
|
||||
/// is the iteration budget, `{baseline}` the pristine baseline binary and
|
||||
/// `{candidate}` the patched one (both repo-root-relative paths, and the
|
||||
/// command runs from the repo root). A fuzz script must never hardcode a
|
||||
/// baseline path — the harness owns where the pristine build lands.
|
||||
fn fuzz_command(tmpl: &str, iters: u64, baseline: &str, candidate: &str) -> String {
|
||||
tmpl.replace("{iters}", &iters.to_string())
|
||||
.replace("{baseline}", baseline)
|
||||
.replace("{candidate}", candidate)
|
||||
}
|
||||
|
||||
/// What the differential-fuzz gate will do, decided before any process runs.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum FuzzPlan {
|
||||
Skip(&'static str),
|
||||
Run(String),
|
||||
}
|
||||
|
||||
/// Differential fuzz is a baseline-vs-candidate gate: it needs both sides.
|
||||
/// Missing either one is a Skip with the reason recorded, never a silent
|
||||
/// self-comparison and never a hard error (the standalone gates flow has
|
||||
/// no baseline by construction).
|
||||
fn plan_fuzz(fuzz: Option<&str>, inputs: &GateInputs) -> FuzzPlan {
|
||||
match (fuzz, inputs.baseline_binary) {
|
||||
(None, _) => FuzzPlan::Skip(FUZZ_SKIP_NO_COMMAND),
|
||||
(Some(_), None) => FuzzPlan::Skip(FUZZ_SKIP_NO_BASELINE),
|
||||
(Some(tmpl), Some(baseline)) => FuzzPlan::Run(fuzz_command(
|
||||
tmpl,
|
||||
inputs.fuzz_iters,
|
||||
baseline,
|
||||
inputs.candidate_binary,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Grade one executed fuzz run. Passed iff the command exited 0, printed a
|
||||
/// well-formed `FUZZ-RESULT` line, and reported zero divergences — a run
|
||||
/// that never reports the line is a Failed gate, not a pass by silence.
|
||||
/// Returns `(outcome, iters_actually_executed, divergence)`.
|
||||
fn fuzz_gate_outcome(
|
||||
stdout: &str,
|
||||
success: bool,
|
||||
exit_code: Option<i32>,
|
||||
) -> (GateOutcome, u64, bool) {
|
||||
match parse_fuzz_result(stdout) {
|
||||
None => (
|
||||
GateOutcome::Failed {
|
||||
detail: "fuzz command did not report FUZZ-RESULT".into(),
|
||||
},
|
||||
0,
|
||||
false,
|
||||
),
|
||||
Some((iters, divergences)) => (
|
||||
if success && divergences == 0 {
|
||||
GateOutcome::Passed
|
||||
} else {
|
||||
GateOutcome::Failed {
|
||||
detail: format!("exit={exit_code:?} iters={iters} divergences={divergences}"),
|
||||
}
|
||||
},
|
||||
iters,
|
||||
divergences > 0,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// The fp-tolerance policy fails closed without a committed reference:
|
||||
/// there is no hash to fall back to that a tolerance could compare against.
|
||||
fn fp_reference(golden_reference: Option<&PathBuf>) -> Result<&PathBuf, GateOutcome> {
|
||||
golden_reference.ok_or_else(|| GateOutcome::Failed {
|
||||
detail: "fp-tolerance policy requires corpus.golden_reference in target.toml".into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// GoldenReplay verdict under an fp-tolerance policy: the replay stdout is
|
||||
/// compared token-wise against the committed reference output.
|
||||
fn fp_golden_outcome(
|
||||
policy: &EquivalencePolicy,
|
||||
reference: &str,
|
||||
stdout: &str,
|
||||
success: bool,
|
||||
exit_code: Option<i32>,
|
||||
) -> GateOutcome {
|
||||
if !success {
|
||||
return GateOutcome::Failed {
|
||||
detail: format!("golden replay command failed (exit={exit_code:?})"),
|
||||
};
|
||||
}
|
||||
match policy.compare(reference, stdout) {
|
||||
Ok(()) => GateOutcome::Passed,
|
||||
Err(d) => GateOutcome::Failed {
|
||||
detail: format!(
|
||||
"diverged at token/offset {}: baseline={:?} candidate={:?} — {}",
|
||||
d.token, d.baseline, d.candidate, d.reason
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the mechanical gates for `target`, testing `inputs.candidate_binary`
|
||||
/// (repo-root-relative) in golden replay and differential fuzz. Differential
|
||||
/// fuzz additionally needs `inputs.baseline_binary` to differ against and is
|
||||
/// skipped-with-reason without one. Pin mismatches (corpus or test suite)
|
||||
/// are errors, not gate failures: nothing runs on tampered inputs.
|
||||
pub fn run_core_gates(
|
||||
root: &Path,
|
||||
spec: &TargetSpec,
|
||||
binary: &str,
|
||||
) -> anyhow::Result<Vec<(GateLayer, GateOutcome)>> {
|
||||
inputs: &GateInputs,
|
||||
) -> anyhow::Result<GateReport> {
|
||||
let verified = pin::verify_manifest(
|
||||
&root.join(&spec.corpus.manifest),
|
||||
&root.join(&spec.corpus.root),
|
||||
@@ -34,6 +185,24 @@ pub fn run_core_gates(
|
||||
spec.corpus.manifest.display()
|
||||
);
|
||||
|
||||
// Test-suite pin: same refuse-on-mismatch posture as the corpus pin,
|
||||
// but the manifest is optional until a human deliberately generates it.
|
||||
let suite_manifest = suite_manifest_path(&spec.corpus.manifest);
|
||||
if root.join(&suite_manifest).exists() {
|
||||
let verified = pin::verify_manifest(&root.join(&suite_manifest), root).map_err(|e| {
|
||||
anyhow::anyhow!("upstream test suite hash mismatch — refusing to run: {e}")
|
||||
})?;
|
||||
eprintln!(
|
||||
"test-suite pin: {verified} files verified against {}",
|
||||
suite_manifest.display()
|
||||
);
|
||||
} else {
|
||||
eprintln!(
|
||||
"test-suite pin: {} not found — suite unpinned (generate with scripts/pin-testsuite.sh)",
|
||||
suite_manifest.display()
|
||||
);
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
let t = sh(root, &spec.gates.tests)?;
|
||||
@@ -51,34 +220,285 @@ pub fn run_core_gates(
|
||||
},
|
||||
));
|
||||
|
||||
let g = sh(root, &spec.gates.golden.replace("{binary}", binary))?;
|
||||
let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
|
||||
let actual = format!("{:x}", Sha256::digest(&g.stdout));
|
||||
results.push((
|
||||
GateLayer::GoldenReplay,
|
||||
if g.status.success() && actual == expected {
|
||||
GateOutcome::Passed
|
||||
} else {
|
||||
GateOutcome::Failed {
|
||||
detail: format!(
|
||||
"exit={:?} expected={expected} actual={actual}",
|
||||
g.status.code()
|
||||
),
|
||||
let policy = EquivalencePolicy::load(&spec.target_dir(root))?;
|
||||
let equivalence_mode = match &policy {
|
||||
EquivalencePolicy::ByteIdentical => "byte-identical",
|
||||
EquivalencePolicy::FpTolerance { .. } => "fp-tolerance",
|
||||
};
|
||||
let golden_cmd = spec
|
||||
.gates
|
||||
.golden
|
||||
.replace("{binary}", inputs.candidate_binary);
|
||||
let golden = match &policy {
|
||||
EquivalencePolicy::ByteIdentical => {
|
||||
let g = sh(root, &golden_cmd)?;
|
||||
let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
|
||||
let actual = format!("{:x}", Sha256::digest(&g.stdout));
|
||||
if g.status.success() && actual == expected {
|
||||
GateOutcome::Passed
|
||||
} else {
|
||||
GateOutcome::Failed {
|
||||
detail: format!(
|
||||
"exit={:?} expected={expected} actual={actual}",
|
||||
g.status.code()
|
||||
),
|
||||
}
|
||||
}
|
||||
},
|
||||
));
|
||||
}
|
||||
EquivalencePolicy::FpTolerance { .. } => {
|
||||
match fp_reference(spec.corpus.golden_reference.as_ref()) {
|
||||
Err(outcome) => outcome,
|
||||
Ok(reference_path) => {
|
||||
let reference =
|
||||
std::fs::read_to_string(root.join(reference_path)).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"cannot read corpus.golden_reference {}: {e}",
|
||||
reference_path.display()
|
||||
)
|
||||
})?;
|
||||
let g = sh(root, &golden_cmd)?;
|
||||
fp_golden_outcome(
|
||||
&policy,
|
||||
&reference,
|
||||
&String::from_utf8_lossy(&g.stdout),
|
||||
g.status.success(),
|
||||
g.status.code(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
results.push((GateLayer::GoldenReplay, golden));
|
||||
|
||||
let (fuzz_outcome, fuzz_iters, fuzz_divergence) =
|
||||
match plan_fuzz(spec.gates.fuzz.as_deref(), inputs) {
|
||||
FuzzPlan::Skip(reason) => (
|
||||
GateOutcome::Skipped {
|
||||
reason: reason.to_string(),
|
||||
},
|
||||
0,
|
||||
false,
|
||||
),
|
||||
FuzzPlan::Run(cmd) => {
|
||||
let f = sh(root, &cmd)?;
|
||||
fuzz_gate_outcome(
|
||||
&String::from_utf8_lossy(&f.stdout),
|
||||
f.status.success(),
|
||||
f.status.code(),
|
||||
)
|
||||
}
|
||||
};
|
||||
results.push((GateLayer::DifferentialFuzz, fuzz_outcome));
|
||||
|
||||
results.push((
|
||||
GateLayer::DifferentialFuzz,
|
||||
GateOutcome::Skipped {
|
||||
reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into(),
|
||||
},
|
||||
));
|
||||
results.push((
|
||||
GateLayer::Sanitizers,
|
||||
GateOutcome::Skipped {
|
||||
reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into(),
|
||||
reason:
|
||||
"sanitizers run on the accept path in verdict (ASan/UBSan, TSan when configured)"
|
||||
.into(),
|
||||
},
|
||||
));
|
||||
Ok(results)
|
||||
|
||||
Ok(GateReport {
|
||||
gates: results,
|
||||
fuzz_iters,
|
||||
fuzz_divergence,
|
||||
equivalence_mode: equivalence_mode.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn suite_manifest_derives_from_corpus_manifest() {
|
||||
assert_eq!(
|
||||
suite_manifest_path(Path::new("corpora/cjson/MANIFEST.sha256")),
|
||||
Path::new("corpora/cjson").join("TESTSUITE.sha256")
|
||||
);
|
||||
assert_eq!(
|
||||
suite_manifest_path(Path::new("MANIFEST.sha256")),
|
||||
Path::new("TESTSUITE.sha256")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_result_parses_last_exact_line() {
|
||||
assert_eq!(
|
||||
parse_fuzz_result("FUZZ-RESULT iters=10000 divergences=0\n"),
|
||||
Some((10000, 0))
|
||||
);
|
||||
// Last matching line wins; trailing non-matching output is ignored.
|
||||
let out = "FUZZ-RESULT iters=1 divergences=1\nprogress...\nFUZZ-RESULT iters=9999 divergences=2\ndone\n";
|
||||
assert_eq!(parse_fuzz_result(out), Some((9999, 2)));
|
||||
// CRLF-terminated capture still parses.
|
||||
assert_eq!(
|
||||
parse_fuzz_result("FUZZ-RESULT iters=42 divergences=0\r\n"),
|
||||
Some((42, 0))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_result_rejects_malformed_lines() {
|
||||
for bad in [
|
||||
"",
|
||||
"no result here",
|
||||
" FUZZ-RESULT iters=1 divergences=0", // leading space
|
||||
"FUZZ-RESULT iters=1 divergences=0", // double space
|
||||
"FUZZ-RESULT iters=abc divergences=0", // non-numeric
|
||||
"FUZZ-RESULT iters=1 divergences=-1", // sign not allowed
|
||||
"FUZZ-RESULT iters=1 divergences=0 xx", // trailing junk
|
||||
"FUZZ-RESULT divergences=0 iters=1", // wrong field order
|
||||
"FUZZ-RESULT iters=1", // missing field
|
||||
] {
|
||||
assert_eq!(parse_fuzz_result(bad), None, "should reject: {bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
fn inputs(baseline: Option<&'static str>) -> GateInputs<'static> {
|
||||
GateInputs {
|
||||
candidate_binary: "targets/cjson/cand/cjson-bench",
|
||||
baseline_binary: baseline,
|
||||
fuzz_iters: 10_000,
|
||||
}
|
||||
}
|
||||
|
||||
const TMPL: &str = "python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work \
|
||||
{iters} {baseline} {candidate}";
|
||||
|
||||
#[test]
|
||||
fn fuzz_command_substitutes_all_three_placeholders() {
|
||||
assert_eq!(
|
||||
fuzz_command(TMPL, 500, "base/bin", "cand/bin"),
|
||||
"python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work 500 base/bin cand/bin"
|
||||
);
|
||||
// Repeated placeholders all substitute; unknown braces are left alone.
|
||||
assert_eq!(
|
||||
fuzz_command("{candidate} {candidate} {iters} {other}", 2, "b", "c"),
|
||||
"c c 2 {other}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_skipped_when_no_command_declared() {
|
||||
assert_eq!(
|
||||
plan_fuzz(None, &inputs(Some("targets/cjson/baseline/cjson-bench"))),
|
||||
FuzzPlan::Skip("no fuzz command declared in target.toml [gates].fuzz")
|
||||
);
|
||||
// Absent command wins even without a baseline: nothing to run either way.
|
||||
assert_eq!(
|
||||
plan_fuzz(None, &inputs(None)),
|
||||
FuzzPlan::Skip(FUZZ_SKIP_NO_COMMAND)
|
||||
);
|
||||
}
|
||||
|
||||
/// The regression this branch exists for: `just gates` (and CI) build only
|
||||
/// the working tree, so there is no pristine baseline. Before the fix the
|
||||
/// fuzz script silently hardcoded targets/<t>/baseline/… — a path nothing
|
||||
/// on that flow builds — and the gate hard-failed. Now it skips, loudly.
|
||||
#[test]
|
||||
fn fuzz_skipped_when_declared_but_no_baseline() {
|
||||
assert_eq!(
|
||||
plan_fuzz(Some(TMPL), &inputs(None)),
|
||||
FuzzPlan::Skip(
|
||||
"differential fuzz needs a pristine baseline binary; run the accept path \
|
||||
(`just verdict`), which rebuilds one"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_runs_with_both_sides() {
|
||||
assert_eq!(
|
||||
plan_fuzz(
|
||||
Some(TMPL),
|
||||
&inputs(Some("targets/cjson/baseline/cjson-bench"))
|
||||
),
|
||||
FuzzPlan::Run(
|
||||
"python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work 10000 \
|
||||
targets/cjson/baseline/cjson-bench targets/cjson/cand/cjson-bench"
|
||||
.to_string()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_gate_grades_clean_run_as_pass() {
|
||||
let (outcome, iters, diverged) =
|
||||
fuzz_gate_outcome("FUZZ-RESULT iters=10000 divergences=0\n", true, Some(0));
|
||||
assert_eq!(outcome, GateOutcome::Passed);
|
||||
// The ledger records iterations *actually executed*, not the budget.
|
||||
assert_eq!(iters, 10_000);
|
||||
assert!(!diverged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_gate_fails_on_divergence_nonzero_exit_or_no_result_line() {
|
||||
// Divergence reported: fail, and the count is still recorded.
|
||||
let (outcome, iters, diverged) =
|
||||
fuzz_gate_outcome("FUZZ-RESULT iters=900 divergences=3\n", false, Some(1));
|
||||
assert!(
|
||||
matches!(outcome, GateOutcome::Failed { detail } if detail.contains("divergences=3"))
|
||||
);
|
||||
assert_eq!(iters, 900);
|
||||
assert!(diverged);
|
||||
|
||||
// Zero divergences but a nonzero exit (crash/timeout) is NOT a pass.
|
||||
let (outcome, ..) =
|
||||
fuzz_gate_outcome("FUZZ-RESULT iters=10 divergences=0\n", false, Some(2));
|
||||
assert!(
|
||||
matches!(outcome, GateOutcome::Failed { detail } if detail.contains("exit=Some(2)"))
|
||||
);
|
||||
|
||||
// A run that never reports the contract line fails closed (e.g. the
|
||||
// script died before printing — the old hardcoded-baseline breakage).
|
||||
let (outcome, iters, diverged) =
|
||||
fuzz_gate_outcome("Traceback (most recent call last):\n", false, Some(1));
|
||||
assert_eq!(
|
||||
outcome,
|
||||
GateOutcome::Failed {
|
||||
detail: "fuzz command did not report FUZZ-RESULT".into()
|
||||
}
|
||||
);
|
||||
assert_eq!(iters, 0);
|
||||
assert!(!diverged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fp_tolerance_requires_golden_reference() {
|
||||
let err = fp_reference(None).unwrap_err();
|
||||
assert_eq!(
|
||||
err,
|
||||
GateOutcome::Failed {
|
||||
detail: "fp-tolerance policy requires corpus.golden_reference in target.toml"
|
||||
.into()
|
||||
}
|
||||
);
|
||||
let path = PathBuf::from("corpora/matmul/GOLDEN.reference");
|
||||
assert_eq!(fp_reference(Some(&path)).unwrap(), &path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fp_golden_outcome_applies_policy() {
|
||||
let policy = EquivalencePolicy::FpTolerance {
|
||||
abs: 1e-6,
|
||||
rel: 1e-5,
|
||||
};
|
||||
// Last-ULP reordering within tolerance passes.
|
||||
assert_eq!(
|
||||
fp_golden_outcome(&policy, "1.0000000 2.0", "1.0000001 2.0", true, Some(0)),
|
||||
GateOutcome::Passed
|
||||
);
|
||||
// A genuine wrong value fails with the divergence in the detail.
|
||||
assert!(matches!(
|
||||
fp_golden_outcome(&policy, "1.0 2.0", "1.0 2.5", true, Some(0)),
|
||||
GateOutcome::Failed { detail } if detail.contains("token/offset 1")
|
||||
));
|
||||
// A failing replay command fails regardless of output.
|
||||
assert!(matches!(
|
||||
fp_golden_outcome(&policy, "1.0", "1.0", false, Some(3)),
|
||||
GateOutcome::Failed { detail } if detail.contains("exit=Some(3)")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,10 @@ pub struct TargetSpec {
|
||||
pub bench: Bench,
|
||||
pub gates: Gates,
|
||||
pub corpus: Corpus,
|
||||
/// Target name (the directory under `targets/`), captured by `load`;
|
||||
/// not part of target.toml itself.
|
||||
#[serde(skip)]
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -52,6 +56,14 @@ pub struct Build {
|
||||
/// Sanitizer binary path with `{out}` substituted.
|
||||
#[serde(default)]
|
||||
pub sanitizer_binary: Option<String>,
|
||||
/// TSan build command; `{out}` substituted. Run on the accept path
|
||||
/// when configured — a patch touching concurrency without a TSan
|
||||
/// lane cannot be machine-accepted.
|
||||
#[serde(default)]
|
||||
pub tsan: Option<String>,
|
||||
/// TSan binary path with `{out}` substituted.
|
||||
#[serde(default)]
|
||||
pub tsan_binary: Option<String>,
|
||||
}
|
||||
|
||||
/// Substitute the `{out}` isolation-dir placeholder in a build command or
|
||||
@@ -66,6 +78,11 @@ pub struct Gates {
|
||||
pub tests: String,
|
||||
/// Golden replay command; stdout is hashed. `{binary}` substituted.
|
||||
pub golden: String,
|
||||
/// Differential-fuzz command; `{candidate}` and `{iters}` substituted.
|
||||
/// Must print `FUZZ-RESULT iters=<n> divergences=<m>` as its final
|
||||
/// stdout line. Absent → the fuzz gate is skipped with a reason.
|
||||
#[serde(default)]
|
||||
pub fuzz: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -77,6 +94,11 @@ pub struct Corpus {
|
||||
/// File whose last whitespace-delimited-first-field line is the
|
||||
/// expected sha256 of the golden command's stdout.
|
||||
pub golden_sha256: PathBuf,
|
||||
/// Committed reference stdout of the golden command, repo-root-
|
||||
/// relative. Required by fp-tolerance targets: tolerance comparison
|
||||
/// needs the actual values, not a hash.
|
||||
#[serde(default)]
|
||||
pub golden_reference: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl TargetSpec {
|
||||
@@ -84,7 +106,15 @@ impl TargetSpec {
|
||||
let path = repo_root.join("targets").join(name).join("target.toml");
|
||||
let raw = std::fs::read_to_string(&path)
|
||||
.map_err(|e| anyhow::anyhow!("cannot read {}: {e}", path.display()))?;
|
||||
Ok(toml::from_str(&raw)?)
|
||||
let mut spec: Self = toml::from_str(&raw)?;
|
||||
spec.name = name.to_string();
|
||||
Ok(spec)
|
||||
}
|
||||
|
||||
/// This target's directory under the repo root (`targets/<name>`),
|
||||
/// where per-target policy files like equivalence.toml live.
|
||||
pub fn target_dir(&self, root: &Path) -> PathBuf {
|
||||
root.join("targets").join(&self.name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,3 +130,48 @@ pub fn expected_golden_hash(path: &Path) -> anyhow::Result<String> {
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| anyhow::anyhow!("no hash entry in {}", path.display()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn load_captures_name_and_defaults_new_fields() {
|
||||
let root = std::env::temp_dir().join("hotpath-target-load");
|
||||
let dir = root.join("targets").join("demo");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(
|
||||
dir.join("target.toml"),
|
||||
r#"
|
||||
[source]
|
||||
repo = "https://example.invalid/demo.git"
|
||||
commit = "deadbeef"
|
||||
license = "MIT"
|
||||
|
||||
[build]
|
||||
baseline = "make -C targets/demo OUT={out}"
|
||||
binary = "{out}/demo"
|
||||
|
||||
[bench]
|
||||
command = "{binary} bench"
|
||||
workload = "demo workload"
|
||||
|
||||
[gates]
|
||||
tests = "make -C targets/demo test"
|
||||
golden = "{binary} replay"
|
||||
|
||||
[corpus]
|
||||
manifest = "corpora/demo/MANIFEST.sha256"
|
||||
root = "corpora/demo"
|
||||
golden_sha256 = "corpora/demo/GOLDEN.sha256"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let spec = TargetSpec::load(&root, "demo").unwrap();
|
||||
assert_eq!(spec.target_dir(&root), root.join("targets").join("demo"));
|
||||
assert!(spec.gates.fuzz.is_none());
|
||||
assert!(spec.corpus.golden_reference.is_none());
|
||||
assert!(spec.build.tsan.is_none());
|
||||
assert!(spec.build.tsan_binary.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "harnessd"
|
||||
description = "Agent-facing IPC daemon: the six-tool surface (SPEC §3.5) over line-delimited JSON on stdio; patches applied only after path-allowlist checks"
|
||||
description = "Agent-facing IPC daemon: the seven-op JSON surface (SPEC §3.5) over line-delimited JSON on stdio; patches applied only after path-allowlist checks"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
//! Diff-header path vetting: the allowlist between agent-authored diffs
|
||||
//! and `git apply` in the target workspace. Portable (no Unix
|
||||
//! dependencies) so the escape-regression tests run on every development
|
||||
//! platform, not just the daemon's POSIX hosts.
|
||||
|
||||
use anyhow::{anyhow, ensure, Result};
|
||||
use std::path::{Component, Path};
|
||||
|
||||
/// Relative paths only; no parent components, no .git, no absolutes.
|
||||
pub fn check_rel_path(p: &str) -> Result<()> {
|
||||
let path = Path::new(p);
|
||||
ensure!(!path.is_absolute(), "absolute paths forbidden");
|
||||
for c in path.components() {
|
||||
match c {
|
||||
Component::Normal(seg) => ensure!(seg != ".git", ".git access forbidden"),
|
||||
Component::CurDir => {}
|
||||
_ => anyhow::bail!("path escapes workspace: {p}"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const RENAME_COPY_HEADERS: [&str; 4] = ["rename from ", "rename to ", "copy from ", "copy to "];
|
||||
|
||||
/// Appended to every `---`/`+++` rejection. That rule is the one place where a
|
||||
/// hunk BODY line can be taken for a header — `-` and `+` are the body markers
|
||||
/// themselves — and the bare error ("lacks the a/ or b/ prefix", "path escapes
|
||||
/// workspace") is baffling if all you did was delete a comment. Say the cause
|
||||
/// out loud. No other header rule can collide: a body line always carries a
|
||||
/// `-`, `+` or space marker, so it can never begin with `diff --git ` or
|
||||
/// `rename `.
|
||||
const HEADER_COLLISION_NOTE: &str = "\
|
||||
note: this line was vetted as a `---`/`+++` file header. The scanner does not track hunk \
|
||||
boundaries — deliberate, and it stays that way: a real header must never slip past it as body \
|
||||
text — so a hunk body line reaches this rule too. REMOVING a line whose own text begins with \
|
||||
`-- ` renders at column 0 as `--- ...` (an unindented SQL/Lua `-- comment`), and ADDING one \
|
||||
that begins with `++ ` renders as `+++ ...`. If that is what this line is, the patch is being \
|
||||
rejected on a false positive. Workaround: leave such lines untouched as context and edit \
|
||||
around them. Rewording or indenting the line does NOT help — the removal itself is what \
|
||||
renders as `--- `. Mind the space: `--x`/`++x` (C decrement/increment) never collide.";
|
||||
|
||||
/// Vet every path-bearing header line of a unified git diff: `diff --git`,
|
||||
/// `---`, `+++`, and the rename/copy extended headers. The whole diff is
|
||||
/// rejected if any extracted path is absolute, contains a `..` component,
|
||||
/// is quoted (git's C-style quoting can encode `..` invisibly), or — for
|
||||
/// `---`/`+++` lines — lacks the standard `a/`/`b/` prefix (`--no-prefix`
|
||||
/// output). Every line is scanned without tracking hunk boundaries, so a
|
||||
/// content line colliding with a header prefix is vetted too: over-rejection
|
||||
/// is the safe direction, and `git apply` remains the backstop.
|
||||
pub fn check_diff_paths(diff: &str) -> Result<()> {
|
||||
for line in diff.lines() {
|
||||
if let Some(rest) = line.strip_prefix("diff --git ") {
|
||||
let (a, b) = split_git_header_paths(rest).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"cannot parse diff header {line:?}: expected \
|
||||
`diff --git a/<path> b/<path>` with plain unquoted paths"
|
||||
)
|
||||
})?;
|
||||
vet_extracted(a)?;
|
||||
vet_extracted(b)?;
|
||||
} else if line.starts_with("--- ") || line.starts_with("+++ ") {
|
||||
vet_file_header(&line[4..]).map_err(|e| {
|
||||
anyhow!("{e}\n offending line: {line:?}\n {HEADER_COLLISION_NOTE}")
|
||||
})?;
|
||||
} else if let Some(rest) = RENAME_COPY_HEADERS
|
||||
.iter()
|
||||
.find_map(|pre| line.strip_prefix(pre))
|
||||
{
|
||||
vet_extracted(header_path(rest))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Vet the path on a `---`/`+++` file-header line; `rest` is the text after
|
||||
/// the four-byte marker. Errors are wrapped by the caller with
|
||||
/// `HEADER_COLLISION_NOTE`, since any line reaching here *might* be a hunk
|
||||
/// body line rather than a header.
|
||||
fn vet_file_header(rest: &str) -> Result<()> {
|
||||
let p = header_path(rest);
|
||||
if p == "/dev/null" {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
!p.starts_with('"'),
|
||||
"quoted diff paths forbidden: {p} (rewrite the patch with plain ASCII paths)"
|
||||
);
|
||||
let stripped = p
|
||||
.strip_prefix("a/")
|
||||
.or_else(|| p.strip_prefix("b/"))
|
||||
.ok_or_else(|| {
|
||||
anyhow!(
|
||||
"diff path {p:?} lacks the a/ or b/ prefix — regenerate with \
|
||||
standard `git diff` output (--no-prefix diffs are rejected)"
|
||||
)
|
||||
})?;
|
||||
vet_extracted(stripped)
|
||||
}
|
||||
|
||||
/// Header paths may carry a traditional-diff `\t<timestamp>` suffix.
|
||||
fn header_path(rest: &str) -> &str {
|
||||
rest.split('\t').next().unwrap_or(rest).trim_end()
|
||||
}
|
||||
|
||||
/// `diff --git a/<x> b/<y>`: split at the LAST ` b/` so a ` b/` embedded
|
||||
/// in <x> cannot truncate <y>; splitting only ever removes the three
|
||||
/// bytes ` b/`, so a `..` component in either real path always survives
|
||||
/// intact into one of the two fragments.
|
||||
fn split_git_header_paths(rest: &str) -> Option<(&str, &str)> {
|
||||
let a_rest = rest.strip_prefix("a/")?;
|
||||
let idx = a_rest.rfind(" b/")?;
|
||||
Some((&a_rest[..idx], a_rest[idx + 3..].trim_end()))
|
||||
}
|
||||
|
||||
fn vet_extracted(p: &str) -> Result<()> {
|
||||
if p == "/dev/null" {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(!p.is_empty(), "empty path in diff header");
|
||||
ensure!(
|
||||
!p.starts_with('"'),
|
||||
"quoted diff paths forbidden: {p} (rewrite the patch with plain ASCII paths)"
|
||||
);
|
||||
ensure!(
|
||||
!p.starts_with('/'),
|
||||
"absolute diff paths forbidden (only /dev/null is allowed, for new/deleted files): {p}"
|
||||
);
|
||||
check_rel_path(p)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rel_path_rules() {
|
||||
assert!(check_rel_path("src/lib.rs").is_ok());
|
||||
assert!(check_rel_path("./src/lib.rs").is_ok());
|
||||
assert!(check_rel_path("../outside").is_err());
|
||||
assert!(check_rel_path("a/../b").is_err());
|
||||
assert!(check_rel_path("/etc/passwd").is_err());
|
||||
assert!(check_rel_path(".git/config").is_err());
|
||||
assert!(check_rel_path("a/.git/hooks").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normal_diff_passes() {
|
||||
let diff = "diff --git a/src/lib.rs b/src/lib.rs\n\
|
||||
index 1111111..2222222 100644\n\
|
||||
--- a/src/lib.rs\n\
|
||||
+++ b/src/lib.rs\n\
|
||||
@@ -1,2 +1,2 @@\n\
|
||||
-old\n\
|
||||
+new\n";
|
||||
assert!(check_diff_paths(diff).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_and_deleted_file_dev_null_passes() {
|
||||
let diff = "diff --git a/newfile.rs b/newfile.rs\n\
|
||||
new file mode 100644\n\
|
||||
--- /dev/null\n\
|
||||
+++ b/newfile.rs\n\
|
||||
@@ -0,0 +1 @@\n\
|
||||
+hello\n\
|
||||
diff --git a/gone.rs b/gone.rs\n\
|
||||
deleted file mode 100644\n\
|
||||
--- a/gone.rs\n\
|
||||
+++ /dev/null\n\
|
||||
@@ -1 +0,0 @@\n\
|
||||
-bye\n";
|
||||
assert!(check_diff_paths(diff).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_workspace_rename_passes() {
|
||||
let diff = "diff --git a/old.rs b/nested/new.rs\n\
|
||||
similarity index 100%\n\
|
||||
rename from old.rs\n\
|
||||
rename to nested/new.rs\n";
|
||||
assert!(check_diff_paths(diff).is_ok());
|
||||
}
|
||||
|
||||
/// The exact fixture from the apply-escape audit: a rename whose
|
||||
/// destination climbs out of the workspace. `---`/`+++` scanning alone
|
||||
/// never sees it — the paths live only in the `diff --git` and
|
||||
/// rename headers.
|
||||
#[test]
|
||||
fn rename_escape_fixture_rejected() {
|
||||
let diff = "diff --git a/src.txt b/../outside/escaped.txt\n\
|
||||
similarity index 100%\n\
|
||||
rename from src.txt\n\
|
||||
rename to ../outside/escaped.txt\n";
|
||||
assert!(check_diff_paths(diff).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_headers_rejected_without_git_header_line() {
|
||||
let diff = "similarity index 100%\n\
|
||||
rename from src.txt\n\
|
||||
rename to ../outside/escaped.txt\n";
|
||||
assert!(check_diff_paths(diff).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_escape_rejected() {
|
||||
let diff = "diff --git a/src.txt b/src2.txt\n\
|
||||
copy from src.txt\n\
|
||||
copy to ../outside/copied.txt\n";
|
||||
assert!(check_diff_paths(diff).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prefix_diff_rejected() {
|
||||
let diff = "diff --git src/lib.rs src/lib.rs\n\
|
||||
index 1111111..2222222 100644\n\
|
||||
--- src/lib.rs\n\
|
||||
+++ src/lib.rs\n\
|
||||
@@ -1 +1 @@\n\
|
||||
-old\n\
|
||||
+new\n";
|
||||
let err = check_diff_paths(diff).unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("diff --git") || err.contains("a/ or b/ prefix"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absolute_path_diff_rejected() {
|
||||
let diff = "diff --git a/x b/x\n\
|
||||
rename from x\n\
|
||||
rename to /etc/passwd\n";
|
||||
assert!(check_diff_paths(diff).is_err());
|
||||
let plus = "--- a/x\n\
|
||||
+++ /etc/passwd\n";
|
||||
assert!(check_diff_paths(plus).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parent_component_in_file_headers_rejected() {
|
||||
assert!(check_diff_paths("--- a/../escape.rs\n+++ b/escape.rs\n").is_err());
|
||||
assert!(check_diff_paths("--- a/ok.rs\n+++ b/nested/../../escape.rs\n").is_err());
|
||||
assert!(check_diff_paths("diff --git a/ok.rs b/../escape.rs\n").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dot_git_path_rejected() {
|
||||
let diff = "diff --git a/.git/hooks/pre-commit b/.git/hooks/pre-commit\n\
|
||||
--- a/.git/hooks/pre-commit\n\
|
||||
+++ b/.git/hooks/pre-commit\n";
|
||||
assert!(check_diff_paths(diff).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quoted_paths_rejected() {
|
||||
assert!(check_diff_paths("diff --git \"a/x\" \"b/y\"\n").is_err());
|
||||
assert!(check_diff_paths("--- \"a/x\"\n+++ b/y\n").is_err());
|
||||
assert!(check_diff_paths("rename to \"\\056\\056/evil\"\n").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_b_slash_cannot_hide_escape() {
|
||||
// Whatever way ` b/` splits this header, the `..` lands in a fragment.
|
||||
assert!(check_diff_paths("diff --git a/x b/y b/../evil\n").is_err());
|
||||
assert!(check_diff_paths("diff --git a/../evil b/y b/z\n").is_err());
|
||||
}
|
||||
|
||||
/// A removed body line that renders as `--- ` at column 0 — deleting an
|
||||
/// unindented SQL/Lua `-- comment` — is still REJECTED (fail-closed is the
|
||||
/// point; tracking hunk boundaries here would risk desyncing from git's own
|
||||
/// parser). But the error must name that cause instead of leaving an agent
|
||||
/// staring at a complaint about a/ and b/ prefixes it never wrote.
|
||||
#[test]
|
||||
fn body_line_colliding_with_header_prefix_explains_itself() {
|
||||
let diff = "diff --git a/q.sql b/q.sql\n\
|
||||
--- a/q.sql\n\
|
||||
+++ b/q.sql\n\
|
||||
@@ -1,2 +1 @@\n\
|
||||
--- drop this comment\n\
|
||||
\x20SELECT 1;\n";
|
||||
let err = check_diff_paths(diff).unwrap_err().to_string();
|
||||
assert!(err.contains("--- drop this comment"), "{err}");
|
||||
assert!(err.contains("hunk body line"), "{err}");
|
||||
assert!(err.contains("context"), "{err}");
|
||||
}
|
||||
|
||||
/// The collision needs the SPACE. A removed `--x;` renders as `---x;` and an
|
||||
/// added `++x;` as `+++x;` — neither is a `--- `/`+++ ` header, and both
|
||||
/// must sail through. Guards against anyone "fixing" the note above by
|
||||
/// widening the prefix match.
|
||||
#[test]
|
||||
fn body_line_without_the_space_does_not_collide() {
|
||||
let diff = "diff --git a/x.c b/x.c\n\
|
||||
--- a/x.c\n\
|
||||
+++ b/x.c\n\
|
||||
@@ -1,2 +1,2 @@\n\
|
||||
---x;\n\
|
||||
+++x;\n";
|
||||
assert!(check_diff_paths(diff).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timestamp_suffix_tolerated() {
|
||||
let diff = "--- a/src/lib.rs\t2026-07-13 00:00:00\n\
|
||||
+++ b/src/lib.rs\t2026-07-13 00:00:00\n";
|
||||
assert!(check_diff_paths(diff).is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
//! The Unix daemon proper: stdio/socket serving, op dispatch, and the
|
||||
//! detached verdict pipeline (setsid + Unix sockets keep this POSIX-only;
|
||||
//! path vetting lives in the portable `allowlist` module).
|
||||
|
||||
use crate::allowlist;
|
||||
use anyhow::{anyhow, ensure, Result};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::{BufRead, Write};
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Serializes mutating ops (git apply, pending writes) across socket
|
||||
/// connections; reads don't need it but the cost is nil at one agent.
|
||||
static HANDLE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
pub fn run() -> Result<()> {
|
||||
let root = std::env::current_dir()?.canonicalize()?;
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if let Some(i) = args.iter().position(|a| a == "--socket") {
|
||||
let path = args
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| anyhow!("--socket requires a path"))?;
|
||||
return serve_socket(&root, Path::new(path));
|
||||
}
|
||||
let stdin = std::io::stdin();
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
for line in stdin.lock().lines() {
|
||||
let line = line?;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let resp = respond(&root, &line);
|
||||
writeln!(stdout, "{resp}")?;
|
||||
stdout.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SPEC §10 socket mode: harnessd runs as the trusted uid and listens on a
|
||||
/// Unix socket; the agent process runs as an unprivileged user whose only
|
||||
/// write path into the repo is this daemon. Socket permissions (owner/group/
|
||||
/// mode) are set by the supervisor script that starts us.
|
||||
fn serve_socket(root: &Path, sock: &Path) -> Result<()> {
|
||||
if let Some(dir) = sock.parent() {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
}
|
||||
let _ = std::fs::remove_file(sock);
|
||||
let listener = UnixListener::bind(sock)?;
|
||||
eprintln!("harnessd listening on {}", sock.display());
|
||||
for conn in listener.incoming() {
|
||||
let stream = match conn {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("accept error: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let root = root.to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let reader = match stream.try_clone() {
|
||||
Ok(s) => std::io::BufReader::new(s),
|
||||
Err(_) => return,
|
||||
};
|
||||
let mut writer = stream;
|
||||
for line in reader.lines() {
|
||||
let Ok(line) = line else { break };
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let resp = respond(&root, &line);
|
||||
if writeln!(writer, "{resp}").is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn respond(root: &Path, line: &str) -> Value {
|
||||
// read_verdict long-polls the ledger (read-only) and must not hold
|
||||
// the lock that serializes mutating ops while it sleeps.
|
||||
let is_read_verdict = serde_json::from_str::<Value>(line)
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.get("op")
|
||||
.and_then(Value::as_str)
|
||||
.map(|o| o == "read_verdict")
|
||||
})
|
||||
.unwrap_or(false);
|
||||
let _guard = if is_read_verdict {
|
||||
None
|
||||
} else {
|
||||
Some(HANDLE_LOCK.lock().unwrap_or_else(|p| p.into_inner()))
|
||||
};
|
||||
match handle(root, line) {
|
||||
Ok(v) => json!({"ok": true, "result": v}),
|
||||
Err(e) => json!({"ok": false, "error": e.to_string()}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-quote a string for safe embedding in a generated shell script.
|
||||
fn shq(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', "'\\''"))
|
||||
}
|
||||
|
||||
/// The exact probe `scripts/no-net.sh` runs before it decides whether to
|
||||
/// isolate: can this host open an unprivileged network namespace? Run it here
|
||||
/// too — same uid, same environment, same host as the wrapper we are about to
|
||||
/// spawn — so the isolation note we record describes the run that actually
|
||||
/// happened instead of the one we hoped for.
|
||||
fn netns_available() -> bool {
|
||||
Command::new("unshare")
|
||||
.args(["--net", "--map-current-user", "true"])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn field<'a>(req: &'a Value, key: &str) -> Result<&'a str> {
|
||||
req.get(key)
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow!("missing string field: {key}"))
|
||||
}
|
||||
|
||||
/// A target name must be a plain directory name — no separators, no dots.
|
||||
fn check_target(name: &str) -> Result<()> {
|
||||
ensure!(
|
||||
!name.is_empty()
|
||||
&& name
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
|
||||
"invalid target name"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const RUNNING_LOCK: &str = "results/pending/RUNNING.lock";
|
||||
|
||||
/// The detached verdict pipeline benches the workspace tree it was launched
|
||||
/// against; HANDLE_LOCK cannot cover it, so a lock file does. Refuse any op
|
||||
/// that would swap that tree while a pipeline is in flight.
|
||||
fn ensure_no_running_pipeline(root: &Path) -> Result<()> {
|
||||
let lock = root.join(RUNNING_LOCK);
|
||||
if lock.exists() {
|
||||
let rid = std::fs::read_to_string(&lock).unwrap_or_default();
|
||||
let rid = rid.trim();
|
||||
let rid = if rid.is_empty() { "unknown" } else { rid };
|
||||
anyhow::bail!(
|
||||
"verdict pipeline {rid} in flight ({RUNNING_LOCK} present — wait for \
|
||||
read_verdict to resolve it, or remove the lock if the pipeline is dead)"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle(root: &Path, line: &str) -> Result<Value> {
|
||||
let req: Value = serde_json::from_str(line)?;
|
||||
match field(&req, "op")? {
|
||||
"read_profile" => {
|
||||
let t = field(&req, "target")?;
|
||||
check_target(t)?;
|
||||
let text = std::fs::read_to_string(root.join(format!("results/{t}/hotspots.txt")))?;
|
||||
Ok(json!({"hotspots_text": text}))
|
||||
}
|
||||
"read_ledger" => {
|
||||
let t = field(&req, "target")?;
|
||||
check_target(t)?;
|
||||
// Read-only surface over the append-only DB.
|
||||
let ledger = ledger::Ledger::open(&root.join("results/ledger.sqlite"))?;
|
||||
let classes = ledger.attempted_classes(t)?;
|
||||
let history = ledger.attempt_history(t)?;
|
||||
Ok(json!({
|
||||
"attempted_playbook_classes": classes,
|
||||
"attempts": history,
|
||||
"total_attempts": ledger.count()?,
|
||||
"note": "a class may be re-entered with a materially NEW hypothesis; \
|
||||
never duplicate a (hotspot, class, hypothesis) that has a verdict",
|
||||
}))
|
||||
}
|
||||
"read_playbook" => {
|
||||
let class = match req.get("class") {
|
||||
Some(Value::Number(n)) => n.as_u64().ok_or_else(|| anyhow!("bad class"))? as u8,
|
||||
Some(Value::String(s)) => s.parse()?,
|
||||
_ => anyhow::bail!("missing class"),
|
||||
};
|
||||
ensure!((1..=7).contains(&class), "playbook class must be 1-7");
|
||||
let dir = root.join("playbook");
|
||||
let entry = std::fs::read_dir(&dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|e| {
|
||||
e.file_name()
|
||||
.to_string_lossy()
|
||||
.starts_with(&format!("{class:02}-"))
|
||||
})
|
||||
.ok_or_else(|| anyhow!("no playbook file for class {class}"))?;
|
||||
Ok(json!({"markdown": std::fs::read_to_string(entry.path())?}))
|
||||
}
|
||||
"read_target_source" => {
|
||||
let t = field(&req, "target")?;
|
||||
let p = field(&req, "path")?;
|
||||
check_target(t)?;
|
||||
allowlist::check_rel_path(p)?;
|
||||
// Serve the PRISTINE (HEAD) content, not the working tree:
|
||||
// propose_patch applies each proposal against a reset
|
||||
// workspace, so reads must show the same base the diff needs
|
||||
// — otherwise reads after a proposal show the patched tree
|
||||
// and the next diff's context lines drift (phase2-comrak-005
|
||||
// burned turns rediscovering this).
|
||||
let ws = root.join(format!("targets/{t}/workspace"));
|
||||
let shown = Command::new("git")
|
||||
.args(["-C", ws.to_str().unwrap(), "show", &format!("HEAD:{p}")])
|
||||
.output()?;
|
||||
ensure!(
|
||||
shown.status.success(),
|
||||
"cannot read {p} at HEAD: {}",
|
||||
String::from_utf8_lossy(&shown.stderr).trim()
|
||||
);
|
||||
let text = String::from_utf8_lossy(&shown.stdout).into_owned();
|
||||
// Optional line window so large source files (100KB+) can be read in
|
||||
// chunks instead of overflowing one response — otherwise the agent
|
||||
// is tempted to reach for a shell it must not have.
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let total = lines.len();
|
||||
let offset = req.get("offset").and_then(Value::as_u64).unwrap_or(0) as usize;
|
||||
let limit = req
|
||||
.get("limit")
|
||||
.and_then(Value::as_u64)
|
||||
.map(|v| v as usize)
|
||||
.unwrap_or(usize::MAX);
|
||||
let end = offset.saturating_add(limit).min(total);
|
||||
let start = offset.min(total);
|
||||
let slice = lines[start..end]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| format!("{}\t{}", start + i + 1, l))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
Ok(json!({
|
||||
"content": slice,
|
||||
"total_lines": total,
|
||||
"returned_lines": [start + 1, end],
|
||||
"truncated": end < total,
|
||||
}))
|
||||
}
|
||||
"propose_patch" => {
|
||||
let t = field(&req, "target")?;
|
||||
let diff = field(&req, "diff")?;
|
||||
let hypothesis = field(&req, "hypothesis")?;
|
||||
check_target(t)?;
|
||||
ensure_no_running_pipeline(root)?;
|
||||
// git rejects a diff without a final newline with an opaque
|
||||
// "corrupt patch" — say it plainly (phase2-comrak-004 burned
|
||||
// several turns rediscovering this).
|
||||
ensure!(
|
||||
diff.ends_with('\n'),
|
||||
"diff must end with a trailing newline (git would report 'corrupt patch' at EOF)"
|
||||
);
|
||||
// Allowlist: every path named by any diff header must be a safe
|
||||
// relative path (the git -C below roots them in the target
|
||||
// workspace; nothing outside it is reachable).
|
||||
allowlist::check_diff_paths(diff)?;
|
||||
let ws = root.join(format!("targets/{t}/workspace"));
|
||||
// Every proposal stands alone: reset the workspace to HEAD
|
||||
// first so probe/iteration patches cannot accumulate into a
|
||||
// tested tree the ledger's patch field doesn't describe
|
||||
// (phase2-comrak-004 audit finding).
|
||||
ensure!(
|
||||
Command::new("git")
|
||||
.args(["-C", ws.to_str().unwrap(), "checkout", "--", "."])
|
||||
.status()?
|
||||
.success(),
|
||||
"workspace reset failed"
|
||||
);
|
||||
ensure!(
|
||||
Command::new("git")
|
||||
.args(["-C", ws.to_str().unwrap(), "clean", "-fdq"])
|
||||
.status()?
|
||||
.success(),
|
||||
"workspace clean failed"
|
||||
);
|
||||
let run_git_apply = |check: bool| -> Result<std::process::Output> {
|
||||
let mut args = vec!["-C", ws.to_str().unwrap(), "apply"];
|
||||
if check {
|
||||
args.push("--check");
|
||||
}
|
||||
args.push("-");
|
||||
let mut child = Command::new("git")
|
||||
.args(&args)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()?;
|
||||
child.stdin.take().unwrap().write_all(diff.as_bytes())?;
|
||||
Ok(child.wait_with_output()?)
|
||||
};
|
||||
let checked = run_git_apply(true)?;
|
||||
ensure!(
|
||||
checked.status.success(),
|
||||
"diff does not apply cleanly against the pristine workspace \
|
||||
(proposals are standalone; prior proposals were reset): {}",
|
||||
String::from_utf8_lossy(&checked.stderr).trim()
|
||||
);
|
||||
let applied = run_git_apply(false)?;
|
||||
ensure!(
|
||||
applied.status.success(),
|
||||
"git apply failed: {}",
|
||||
String::from_utf8_lossy(&applied.stderr).trim()
|
||||
);
|
||||
let patch_id = format!("{:x}", Sha256::digest(diff.as_bytes()))[..12].to_string();
|
||||
let pending = root.join("results/pending");
|
||||
std::fs::create_dir_all(&pending)?;
|
||||
std::fs::write(
|
||||
pending.join(format!("{patch_id}.json")),
|
||||
serde_json::to_string_pretty(
|
||||
&json!({"target": t, "hypothesis": hypothesis, "diff": diff}),
|
||||
)?,
|
||||
)?;
|
||||
Ok(json!({"patch_id": patch_id}))
|
||||
}
|
||||
"run_verdict" => {
|
||||
let patch_id = field(&req, "patch_id")?;
|
||||
ensure!(
|
||||
patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12,
|
||||
"bad patch id"
|
||||
);
|
||||
let run_id = field(&req, "run_id")?;
|
||||
ensure!(
|
||||
run_id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
|
||||
"bad run id"
|
||||
);
|
||||
let class = field(&req, "playbook_class")?;
|
||||
let hotspot = field(&req, "hotspot")?;
|
||||
ensure_no_running_pipeline(root)?;
|
||||
let pending: Value = serde_json::from_str(&std::fs::read_to_string(
|
||||
root.join(format!("results/pending/{patch_id}.json")),
|
||||
)?)?;
|
||||
let t = pending["target"].as_str().unwrap();
|
||||
let diff = pending["diff"].as_str().unwrap();
|
||||
// Defense in depth: the pending file sat on disk since
|
||||
// propose_patch vetted it — vet again before applying.
|
||||
allowlist::check_diff_paths(diff)?;
|
||||
let spec = diff_test::target::TargetSpec::load(root, t)?;
|
||||
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
||||
std::fs::write(&diff_path, diff)?;
|
||||
// Re-establish the exact tree the pipeline must test: proposals
|
||||
// after this one may have reset/repatched the workspace.
|
||||
let ws = root.join(format!("targets/{t}/workspace"));
|
||||
let git_ws = |args: &[&str]| -> Result<()> {
|
||||
let out = Command::new("git").arg("-C").arg(&ws).args(args).output()?;
|
||||
ensure!(
|
||||
out.status.success(),
|
||||
"git {} failed in target workspace: {}",
|
||||
args.join(" "),
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
Ok(())
|
||||
};
|
||||
git_ws(&["checkout", "--", "."])?;
|
||||
git_ws(&["clean", "-fdq"])?;
|
||||
let dp = diff_path.to_str().unwrap();
|
||||
git_ws(&["apply", "--check", dp]).map_err(|e| {
|
||||
anyhow!(
|
||||
"pending diff {patch_id} no longer applies to the pristine \
|
||||
workspace — re-propose it: {e}"
|
||||
)
|
||||
})?;
|
||||
git_ws(&["apply", dp])?;
|
||||
// The build + verdict pipeline runs for minutes — far past the MCP
|
||||
// transport's per-call cap. Launch it detached, writing progress to
|
||||
// a log; the agent observes the result via the read_verdict op,
|
||||
// which reads the append-only ledger row once written.
|
||||
// {out} isolates the candidate build (cargo: CARGO_TARGET_DIR;
|
||||
// make: build+copy into it) — language-agnostic, mirrors the
|
||||
// baseline rebuild in verdict.
|
||||
let cand_dir = root
|
||||
.join(format!("targets/{t}/candidate-{patch_id}"))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let cand_build = diff_test::target::subst_out(&spec.build.baseline, &cand_dir);
|
||||
let cand_bin = diff_test::target::subst_out(&spec.build.binary, &cand_dir);
|
||||
let log = format!("results/pending/{run_id}.log");
|
||||
// Target code must not reach the network: the pipeline launches
|
||||
// through a wrapper (default scripts/no-net.sh, a network-namespace
|
||||
// exec); HOTPATH_VERDICT_WRAPPER overrides with a single executable
|
||||
// path for hosts where unshare is unavailable.
|
||||
let (wrapper, isolation_note) = match std::env::var("HOTPATH_VERDICT_WRAPPER") {
|
||||
Ok(w) => {
|
||||
let note = w.clone();
|
||||
(w, note)
|
||||
}
|
||||
Err(_) => {
|
||||
let w = root
|
||||
.join("scripts/no-net.sh")
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
// Honesty: this note is echoed to the agent AND stamped into
|
||||
// the ledger's env fingerprint. With HOTPATH_ALLOW_UNISOLATED=1
|
||||
// set, no-net.sh execs the pipeline with FULL network access on
|
||||
// any host where it cannot open a netns — recording a bare
|
||||
// "no-net.sh" there would claim an isolation the run never had.
|
||||
// Probe the same condition the wrapper will test, and say which
|
||||
// of the two actually applies.
|
||||
let allow_unisolated = std::env::var("HOTPATH_ALLOW_UNISOLATED")
|
||||
.map(|v| v == "1")
|
||||
.unwrap_or(false);
|
||||
let note = if allow_unisolated && !netns_available() {
|
||||
"no-net.sh (HOTPATH_ALLOW_UNISOLATED=1 — network NOT isolated)"
|
||||
} else {
|
||||
"no-net.sh"
|
||||
};
|
||||
(w, note.to_string())
|
||||
}
|
||||
};
|
||||
// Two scripts, because failure accounting has to survive a wrapper
|
||||
// that never execs its argv.
|
||||
//
|
||||
// INNER — runs inside the isolation wrapper. The guarded section is
|
||||
// a SUBSHELL, not the left operand of an `&&`: POSIX sh ignores
|
||||
// `set -e` inside any compound command that is an AND-OR operand
|
||||
// other than the last, so the previous
|
||||
// { set -e; <build>; verdict; } && rm -f lock || { ...; }
|
||||
// shape left errexit inert. A failing candidate build fell straight
|
||||
// through to `cargo run -p verdict`, which then benched — and wrote
|
||||
// a ledger row for — a STALE or missing candidate binary. That is
|
||||
// the exact class of false evidence this project exists to prevent.
|
||||
// A standalone subshell scopes errexit and makes it live again, so a
|
||||
// failed build aborts before verdict can measure anything. The
|
||||
// subshell is the script's last command, so its status is the
|
||||
// script's status.
|
||||
let inner = format!(
|
||||
"printf %s {rid} > {lock}\n\
|
||||
( set -e\n{build}\ncargo run -q -p verdict -- {tgt} \
|
||||
--rebuild-baseline --candidate-bin {cbin} --run-id {rid} --playbook-class {cls} \
|
||||
--hypothesis {hyp} --hotspot {hs} --patch-file {pf} --isolation-note {iso}\n\
|
||||
)\n",
|
||||
rid = shq(run_id),
|
||||
lock = shq(RUNNING_LOCK),
|
||||
build = cand_build,
|
||||
tgt = shq(t),
|
||||
cbin = shq(&cand_bin),
|
||||
cls = shq(class),
|
||||
hyp = shq(pending["hypothesis"].as_str().unwrap()),
|
||||
hs = shq(hotspot),
|
||||
pf = shq(diff_path.to_str().unwrap()),
|
||||
iso = shq(&isolation_note),
|
||||
);
|
||||
// OUTER — runs OUTSIDE the wrapper: invoke the wrapper, then account
|
||||
// for its exit status no matter what it did. scripts/no-net.sh exits
|
||||
// 97 WITHOUT exec'ing its argv when unprivileged netns are
|
||||
// unavailable, and a missing or non-executable HOTPATH_VERDICT_WRAPPER
|
||||
// exits 127/126 — in all three cases the inner script never runs at
|
||||
// all. While this accounting lived inside the wrapper, those cases
|
||||
// removed no lock and wrote no marker: read_verdict polled "running"
|
||||
// forever and every later propose_patch/run_verdict was refused with
|
||||
// "verdict pipeline in flight" until a human deleted the lock by hand.
|
||||
// Out here it always runs, so the run is reported failed and the
|
||||
// daemon stays usable. The wrapper's own stderr (no-net.sh's refusal
|
||||
// text) is already captured in the run log.
|
||||
let script = format!(
|
||||
"{wrapper} sh -c {inner}\n\
|
||||
rc=$?\n\
|
||||
rm -f {lock}\n\
|
||||
[ \"$rc\" -eq 0 ] || echo \"HOTPATH-PIPELINE-FAILED (exit $rc)\"\n",
|
||||
wrapper = shq(&wrapper),
|
||||
inner = shq(&inner),
|
||||
lock = shq(RUNNING_LOCK),
|
||||
);
|
||||
let logf = std::fs::File::create(root.join(&log))?;
|
||||
let errf = logf.try_clone()?;
|
||||
// Write the lock before spawning: the detached script's own
|
||||
// write lands only after exec latency, and a propose_patch in
|
||||
// that window would swap the tree under the pipeline.
|
||||
let lock_path = root.join(RUNNING_LOCK);
|
||||
std::fs::write(&lock_path, run_id)?;
|
||||
// The wrapper is invoked BY the outer script (and is shq'd into it),
|
||||
// not by setsid — that is what puts the accounting outside it.
|
||||
let spawned = Command::new("setsid")
|
||||
.arg("sh")
|
||||
.arg("-c")
|
||||
.arg(&script)
|
||||
.current_dir(root)
|
||||
.stdout(std::process::Stdio::from(logf))
|
||||
.stderr(std::process::Stdio::from(errf))
|
||||
.spawn();
|
||||
if let Err(e) = spawned {
|
||||
let _ = std::fs::remove_file(&lock_path);
|
||||
return Err(anyhow!("failed to launch verdict pipeline: {e}"));
|
||||
}
|
||||
Ok(json!({"status": "started", "run_id": run_id, "log": log,
|
||||
"isolation": isolation_note,
|
||||
"note": "pipeline runs detached; poll read_verdict with this run_id"}))
|
||||
}
|
||||
"read_verdict" => {
|
||||
let run_id = field(&req, "run_id")?;
|
||||
// Long-poll: the pipeline runs for minutes and the MCP transport
|
||||
// caps a tool call at 60s. Waiting ~45s server-side per poll cuts
|
||||
// the agent's turn burn ~15x versus instant "running" replies.
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(45);
|
||||
loop {
|
||||
let ledger = ledger::Ledger::open(&root.join("results/ledger.sqlite"))?;
|
||||
if let Some(v) = ledger.verdict_summary(run_id)? {
|
||||
return Ok(v);
|
||||
}
|
||||
drop(ledger);
|
||||
// A pipeline that died before the verdict binary ran never
|
||||
// writes a row; the launcher's failure accounting — which runs
|
||||
// outside the isolation wrapper, so it survives a wrapper that
|
||||
// refuses to exec — marks the log instead.
|
||||
if run_id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
|
||||
{
|
||||
let log = root.join(format!("results/pending/{run_id}.log"));
|
||||
if let Ok(text) = std::fs::read_to_string(&log) {
|
||||
if text.contains("HOTPATH-PIPELINE-FAILED") {
|
||||
let tail: String = text
|
||||
.lines()
|
||||
.rev()
|
||||
.take(12)
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
return Ok(json!({
|
||||
"status": "failed",
|
||||
"note": "the pipeline died before producing a verdict — usually a \
|
||||
candidate build failure, otherwise a crash or the isolation \
|
||||
wrapper refusing to run (scripts/no-net.sh exits 97 when it \
|
||||
cannot open a network namespace). The marker line carries the \
|
||||
exit status. No ledger row was or will be written for this \
|
||||
run_id, and the run lock has been released. Log tail follows.",
|
||||
"log_tail": tail,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
return Ok(json!({"status": "running",
|
||||
"note": "no ledger row yet; the pipeline is still building/benching — poll again"}));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
}
|
||||
}
|
||||
other => Err(anyhow!(
|
||||
"unknown op: {other} (seven ops, SPEC §3.5 + async read_verdict)"
|
||||
)),
|
||||
}
|
||||
}
|
||||
+22
-431
@@ -1,438 +1,29 @@
|
||||
//! harnessd — the only door between the agent and the trust layer
|
||||
//! (SPEC §3.5, §10). One JSON request per stdin line, one JSON response
|
||||
//! per stdout line. The agent gets exactly six operations; everything
|
||||
//! else (shell, trust-layer writes, patches outside the target
|
||||
//! workspace) simply has no code path here. Phase 1 caveat, recorded:
|
||||
//! same-uid filesystem read-only enforcement still requires the
|
||||
//! separate-user/container setup (Phase 1 infra gap).
|
||||
//! per stdout line. The agent gets exactly seven operations (the six
|
||||
//! synchronous ops plus the async read_verdict poll); everything else
|
||||
//! (shell, trust-layer writes, patches outside the target workspace)
|
||||
//! simply has no code path here. Phase 1 caveat, recorded: same-uid
|
||||
//! filesystem read-only enforcement still requires the separate-user/
|
||||
//! container setup (Phase 1 infra gap).
|
||||
//!
|
||||
//! The daemon itself is POSIX-only (Unix sockets, setsid); the diff
|
||||
//! allowlist is a portable module so its escape-regression tests run on
|
||||
//! every development platform.
|
||||
|
||||
use anyhow::{anyhow, ensure, Result};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::{BufRead, Write};
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::path::{Component, Path};
|
||||
use std::process::Command;
|
||||
use std::sync::Mutex;
|
||||
// On non-Unix only the allowlist tests compile against this module.
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
mod allowlist;
|
||||
#[cfg(unix)]
|
||||
mod daemon;
|
||||
|
||||
/// Serializes mutating ops (git apply, pending writes) across socket
|
||||
/// connections; reads don't need it but the cost is nil at one agent.
|
||||
static HANDLE_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let root = std::env::current_dir()?.canonicalize()?;
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if let Some(i) = args.iter().position(|a| a == "--socket") {
|
||||
let path = args
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| anyhow!("--socket requires a path"))?;
|
||||
return serve_socket(&root, Path::new(path));
|
||||
}
|
||||
let stdin = std::io::stdin();
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
for line in stdin.lock().lines() {
|
||||
let line = line?;
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let resp = respond(&root, &line);
|
||||
writeln!(stdout, "{resp}")?;
|
||||
stdout.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
#[cfg(unix)]
|
||||
fn main() -> anyhow::Result<()> {
|
||||
daemon::run()
|
||||
}
|
||||
|
||||
/// SPEC §10 socket mode: harnessd runs as the trusted uid and listens on a
|
||||
/// Unix socket; the agent process runs as an unprivileged user whose only
|
||||
/// write path into the repo is this daemon. Socket permissions (owner/group/
|
||||
/// mode) are set by the supervisor script that starts us.
|
||||
fn serve_socket(root: &Path, sock: &Path) -> Result<()> {
|
||||
if let Some(dir) = sock.parent() {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
}
|
||||
let _ = std::fs::remove_file(sock);
|
||||
let listener = UnixListener::bind(sock)?;
|
||||
eprintln!("harnessd listening on {}", sock.display());
|
||||
for conn in listener.incoming() {
|
||||
let stream = match conn {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("accept error: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let root = root.to_path_buf();
|
||||
std::thread::spawn(move || {
|
||||
let reader = match stream.try_clone() {
|
||||
Ok(s) => std::io::BufReader::new(s),
|
||||
Err(_) => return,
|
||||
};
|
||||
let mut writer = stream;
|
||||
for line in reader.lines() {
|
||||
let Ok(line) = line else { break };
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let resp = respond(&root, &line);
|
||||
if writeln!(writer, "{resp}").is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn respond(root: &Path, line: &str) -> Value {
|
||||
// read_verdict long-polls the ledger (read-only) and must not hold
|
||||
// the lock that serializes mutating ops while it sleeps.
|
||||
let is_read_verdict = serde_json::from_str::<Value>(line)
|
||||
.ok()
|
||||
.and_then(|v| {
|
||||
v.get("op")
|
||||
.and_then(Value::as_str)
|
||||
.map(|o| o == "read_verdict")
|
||||
})
|
||||
.unwrap_or(false);
|
||||
let _guard = if is_read_verdict {
|
||||
None
|
||||
} else {
|
||||
Some(HANDLE_LOCK.lock().unwrap_or_else(|p| p.into_inner()))
|
||||
};
|
||||
match handle(root, line) {
|
||||
Ok(v) => json!({"ok": true, "result": v}),
|
||||
Err(e) => json!({"ok": false, "error": e.to_string()}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-quote a string for safe embedding in a generated shell script.
|
||||
fn shq(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', "'\\''"))
|
||||
}
|
||||
|
||||
fn field<'a>(req: &'a Value, key: &str) -> Result<&'a str> {
|
||||
req.get(key)
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow!("missing string field: {key}"))
|
||||
}
|
||||
|
||||
/// A target name must be a plain directory name — no separators, no dots.
|
||||
fn check_target(name: &str) -> Result<()> {
|
||||
ensure!(
|
||||
!name.is_empty()
|
||||
&& name
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
|
||||
"invalid target name"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Relative paths only; no parent components, no .git, no absolutes.
|
||||
fn check_rel_path(p: &str) -> Result<()> {
|
||||
let path = Path::new(p);
|
||||
ensure!(!path.is_absolute(), "absolute paths forbidden");
|
||||
for c in path.components() {
|
||||
match c {
|
||||
Component::Normal(seg) => ensure!(seg != ".git", ".git access forbidden"),
|
||||
Component::CurDir => {}
|
||||
_ => anyhow::bail!("path escapes workspace: {p}"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle(root: &Path, line: &str) -> Result<Value> {
|
||||
let req: Value = serde_json::from_str(line)?;
|
||||
match field(&req, "op")? {
|
||||
"read_profile" => {
|
||||
let t = field(&req, "target")?;
|
||||
check_target(t)?;
|
||||
let text = std::fs::read_to_string(root.join(format!("results/{t}/hotspots.txt")))?;
|
||||
Ok(json!({"hotspots_text": text}))
|
||||
}
|
||||
"read_ledger" => {
|
||||
let t = field(&req, "target")?;
|
||||
check_target(t)?;
|
||||
// Read-only surface over the append-only DB.
|
||||
let ledger = ledger::Ledger::open(&root.join("results/ledger.sqlite"))?;
|
||||
let classes = ledger.attempted_classes(t)?;
|
||||
let history = ledger.attempt_history(t)?;
|
||||
Ok(json!({
|
||||
"attempted_playbook_classes": classes,
|
||||
"attempts": history,
|
||||
"total_attempts": ledger.count()?,
|
||||
"note": "a class may be re-entered with a materially NEW hypothesis; \
|
||||
never duplicate a (hotspot, class, hypothesis) that has a verdict",
|
||||
}))
|
||||
}
|
||||
"read_playbook" => {
|
||||
let class = match req.get("class") {
|
||||
Some(Value::Number(n)) => n.as_u64().ok_or_else(|| anyhow!("bad class"))? as u8,
|
||||
Some(Value::String(s)) => s.parse()?,
|
||||
_ => anyhow::bail!("missing class"),
|
||||
};
|
||||
ensure!((1..=7).contains(&class), "playbook class must be 1-7");
|
||||
let dir = root.join("playbook");
|
||||
let entry = std::fs::read_dir(&dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|e| {
|
||||
e.file_name()
|
||||
.to_string_lossy()
|
||||
.starts_with(&format!("{class:02}-"))
|
||||
})
|
||||
.ok_or_else(|| anyhow!("no playbook file for class {class}"))?;
|
||||
Ok(json!({"markdown": std::fs::read_to_string(entry.path())?}))
|
||||
}
|
||||
"read_target_source" => {
|
||||
let t = field(&req, "target")?;
|
||||
let p = field(&req, "path")?;
|
||||
check_target(t)?;
|
||||
check_rel_path(p)?;
|
||||
// Serve the PRISTINE (HEAD) content, not the working tree:
|
||||
// propose_patch applies each proposal against a reset
|
||||
// workspace, so reads must show the same base the diff needs
|
||||
// — otherwise reads after a proposal show the patched tree
|
||||
// and the next diff's context lines drift (phase2-comrak-005
|
||||
// burned turns rediscovering this).
|
||||
let ws = root.join(format!("targets/{t}/workspace"));
|
||||
let shown = Command::new("git")
|
||||
.args(["-C", ws.to_str().unwrap(), "show", &format!("HEAD:{p}")])
|
||||
.output()?;
|
||||
ensure!(
|
||||
shown.status.success(),
|
||||
"cannot read {p} at HEAD: {}",
|
||||
String::from_utf8_lossy(&shown.stderr).trim()
|
||||
);
|
||||
let text = String::from_utf8_lossy(&shown.stdout).into_owned();
|
||||
// Optional line window so large source files (100KB+) can be read in
|
||||
// chunks instead of overflowing one response — otherwise the agent
|
||||
// is tempted to reach for a shell it must not have.
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let total = lines.len();
|
||||
let offset = req.get("offset").and_then(Value::as_u64).unwrap_or(0) as usize;
|
||||
let limit = req
|
||||
.get("limit")
|
||||
.and_then(Value::as_u64)
|
||||
.map(|v| v as usize)
|
||||
.unwrap_or(usize::MAX);
|
||||
let end = offset.saturating_add(limit).min(total);
|
||||
let start = offset.min(total);
|
||||
let slice = lines[start..end]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| format!("{}\t{}", start + i + 1, l))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
Ok(json!({
|
||||
"content": slice,
|
||||
"total_lines": total,
|
||||
"returned_lines": [start + 1, end],
|
||||
"truncated": end < total,
|
||||
}))
|
||||
}
|
||||
"propose_patch" => {
|
||||
let t = field(&req, "target")?;
|
||||
let diff = field(&req, "diff")?;
|
||||
let hypothesis = field(&req, "hypothesis")?;
|
||||
check_target(t)?;
|
||||
// git rejects a diff without a final newline with an opaque
|
||||
// "corrupt patch" — say it plainly (phase2-comrak-004 burned
|
||||
// several turns rediscovering this).
|
||||
ensure!(
|
||||
diff.ends_with('\n'),
|
||||
"diff must end with a trailing newline (git would report 'corrupt patch' at EOF)"
|
||||
);
|
||||
// Allowlist: every path named by the diff must be a safe
|
||||
// relative path (the git -C below roots them in the target
|
||||
// workspace; nothing outside it is reachable). `/dev/null`
|
||||
// is the one absolute path git legitimately emits, for
|
||||
// new/deleted files.
|
||||
for l in diff.lines() {
|
||||
if let Some(p) = l
|
||||
.strip_prefix("--- a/")
|
||||
.or_else(|| l.strip_prefix("+++ b/"))
|
||||
{
|
||||
check_rel_path(p.trim())?;
|
||||
}
|
||||
ensure!(
|
||||
(!l.starts_with("--- /") && !l.starts_with("+++ /"))
|
||||
|| l == "--- /dev/null"
|
||||
|| l == "+++ /dev/null",
|
||||
"absolute diff paths forbidden (only /dev/null is allowed, for new/deleted files)"
|
||||
);
|
||||
}
|
||||
let ws = root.join(format!("targets/{t}/workspace"));
|
||||
// Every proposal stands alone: reset the workspace to HEAD
|
||||
// first so probe/iteration patches cannot accumulate into a
|
||||
// tested tree the ledger's patch field doesn't describe
|
||||
// (phase2-comrak-004 audit finding).
|
||||
ensure!(
|
||||
Command::new("git")
|
||||
.args(["-C", ws.to_str().unwrap(), "checkout", "--", "."])
|
||||
.status()?
|
||||
.success(),
|
||||
"workspace reset failed"
|
||||
);
|
||||
ensure!(
|
||||
Command::new("git")
|
||||
.args(["-C", ws.to_str().unwrap(), "clean", "-fdq"])
|
||||
.status()?
|
||||
.success(),
|
||||
"workspace clean failed"
|
||||
);
|
||||
let run_git_apply = |check: bool| -> Result<std::process::Output> {
|
||||
let mut args = vec!["-C", ws.to_str().unwrap(), "apply"];
|
||||
if check {
|
||||
args.push("--check");
|
||||
}
|
||||
args.push("-");
|
||||
let mut child = Command::new("git")
|
||||
.args(&args)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()?;
|
||||
child.stdin.take().unwrap().write_all(diff.as_bytes())?;
|
||||
Ok(child.wait_with_output()?)
|
||||
};
|
||||
let checked = run_git_apply(true)?;
|
||||
ensure!(
|
||||
checked.status.success(),
|
||||
"diff does not apply cleanly against the pristine workspace \
|
||||
(proposals are standalone; prior proposals were reset): {}",
|
||||
String::from_utf8_lossy(&checked.stderr).trim()
|
||||
);
|
||||
let applied = run_git_apply(false)?;
|
||||
ensure!(
|
||||
applied.status.success(),
|
||||
"git apply failed: {}",
|
||||
String::from_utf8_lossy(&applied.stderr).trim()
|
||||
);
|
||||
let patch_id = format!("{:x}", Sha256::digest(diff.as_bytes()))[..12].to_string();
|
||||
let pending = root.join("results/pending");
|
||||
std::fs::create_dir_all(&pending)?;
|
||||
std::fs::write(
|
||||
pending.join(format!("{patch_id}.json")),
|
||||
serde_json::to_string_pretty(
|
||||
&json!({"target": t, "hypothesis": hypothesis, "diff": diff}),
|
||||
)?,
|
||||
)?;
|
||||
Ok(json!({"patch_id": patch_id}))
|
||||
}
|
||||
"run_verdict" => {
|
||||
let patch_id = field(&req, "patch_id")?;
|
||||
ensure!(
|
||||
patch_id.chars().all(|c| c.is_ascii_hexdigit()) && patch_id.len() == 12,
|
||||
"bad patch id"
|
||||
);
|
||||
let run_id = field(&req, "run_id")?;
|
||||
ensure!(
|
||||
run_id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
|
||||
"bad run id"
|
||||
);
|
||||
let class = field(&req, "playbook_class")?;
|
||||
let hotspot = field(&req, "hotspot")?;
|
||||
let pending: Value = serde_json::from_str(&std::fs::read_to_string(
|
||||
root.join(format!("results/pending/{patch_id}.json")),
|
||||
)?)?;
|
||||
let t = pending["target"].as_str().unwrap();
|
||||
let spec = diff_test::target::TargetSpec::load(root, t)?;
|
||||
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
||||
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
|
||||
// The build + verdict pipeline runs for minutes — far past the MCP
|
||||
// transport's per-call cap. Launch it detached, writing progress to
|
||||
// a log; the agent observes the result via the read_verdict op,
|
||||
// which reads the append-only ledger row once written.
|
||||
// {out} isolates the candidate build (cargo: CARGO_TARGET_DIR;
|
||||
// make: build+copy into it) — language-agnostic, mirrors the
|
||||
// baseline rebuild in verdict.
|
||||
let cand_dir = root
|
||||
.join(format!("targets/{t}/candidate-{patch_id}"))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let cand_build = diff_test::target::subst_out(&spec.build.baseline, &cand_dir);
|
||||
let cand_bin = diff_test::target::subst_out(&spec.build.binary, &cand_dir);
|
||||
let log = format!("results/pending/{run_id}.log");
|
||||
// On any failure (candidate build, gates crash) the trap line
|
||||
// marks the log so read_verdict can report "failed" instead of
|
||||
// leaving the agent polling a verdict that will never exist.
|
||||
let script = format!(
|
||||
"{{ set -e\n{build}\ncargo run -q -p verdict -- {tgt} \
|
||||
--rebuild-baseline --candidate-bin {cbin} --run-id {rid} --playbook-class {cls} \
|
||||
--hypothesis {hyp} --hotspot {hs} --patch-file {pf}\n}} || echo HOTPATH-PIPELINE-FAILED\n",
|
||||
build = cand_build,
|
||||
tgt = shq(t),
|
||||
cbin = shq(&cand_bin),
|
||||
rid = shq(run_id),
|
||||
cls = shq(class),
|
||||
hyp = shq(pending["hypothesis"].as_str().unwrap()),
|
||||
hs = shq(hotspot),
|
||||
pf = shq(diff_path.to_str().unwrap()),
|
||||
);
|
||||
let logf = std::fs::File::create(root.join(&log))?;
|
||||
let errf = logf.try_clone()?;
|
||||
Command::new("setsid")
|
||||
.arg("sh")
|
||||
.arg("-c")
|
||||
.arg(&script)
|
||||
.current_dir(root)
|
||||
.stdout(std::process::Stdio::from(logf))
|
||||
.stderr(std::process::Stdio::from(errf))
|
||||
.spawn()?;
|
||||
Ok(json!({"status": "started", "run_id": run_id, "log": log,
|
||||
"note": "pipeline runs detached; poll read_verdict with this run_id"}))
|
||||
}
|
||||
"read_verdict" => {
|
||||
let run_id = field(&req, "run_id")?;
|
||||
// Long-poll: the pipeline runs for minutes and the MCP transport
|
||||
// caps a tool call at 60s. Waiting ~45s server-side per poll cuts
|
||||
// the agent's turn burn ~15x versus instant "running" replies.
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(45);
|
||||
loop {
|
||||
let ledger = ledger::Ledger::open(&root.join("results/ledger.sqlite"))?;
|
||||
if let Some(v) = ledger.verdict_summary(run_id)? {
|
||||
return Ok(v);
|
||||
}
|
||||
drop(ledger);
|
||||
// A pipeline that died before the verdict binary ran never
|
||||
// writes a row; its trap line marks the log instead.
|
||||
if run_id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
|
||||
{
|
||||
let log = root.join(format!("results/pending/{run_id}.log"));
|
||||
if let Ok(text) = std::fs::read_to_string(&log) {
|
||||
if text.contains("HOTPATH-PIPELINE-FAILED") {
|
||||
let tail: String = text
|
||||
.lines()
|
||||
.rev()
|
||||
.take(12)
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
return Ok(json!({
|
||||
"status": "failed",
|
||||
"note": "the pipeline died before producing a verdict (usually a \
|
||||
candidate build failure); no ledger row was or will be \
|
||||
written for this run_id. Failure log tail follows.",
|
||||
"log_tail": tail,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
return Ok(json!({"status": "running",
|
||||
"note": "no ledger row yet; the pipeline is still building/benching — poll again"}));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
}
|
||||
}
|
||||
other => Err(anyhow!(
|
||||
"unknown op: {other} (seven ops, SPEC §3.5 + async read_verdict)"
|
||||
)),
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
fn main() {
|
||||
eprintln!("harnessd requires a Unix host (Unix sockets, setsid)");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
@@ -36,7 +36,10 @@ impl Verdict {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gate outcomes for one attempt (SPEC §3.2).
|
||||
/// Gate outcomes for one attempt (SPEC §3.2). The trailing fields are
|
||||
/// defaulted so historical rows (written before they existed) keep
|
||||
/// deserializing; they live only in the `gates` JSON column — no schema
|
||||
/// change, append-only doctrine untouched.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct GateResults {
|
||||
pub upstream_tests: bool,
|
||||
@@ -44,6 +47,15 @@ pub struct GateResults {
|
||||
pub fuzz_iters: u64,
|
||||
pub fuzz_divergence: bool,
|
||||
pub sanitizers_clean: bool,
|
||||
/// None = no TSan lane configured/run for this attempt.
|
||||
#[serde(default)]
|
||||
pub tsan_clean: Option<bool>,
|
||||
/// Lexical NHR-classifier hits (crates/verdict src/risk.rs).
|
||||
#[serde(default)]
|
||||
pub risk_signals: Vec<String>,
|
||||
/// "byte-identical" | "fp-tolerance"; None for historical rows.
|
||||
#[serde(default)]
|
||||
pub equivalence_mode: Option<String>,
|
||||
}
|
||||
|
||||
/// Benchmark evidence for one attempt. All times in seconds; CIs are
|
||||
@@ -334,6 +346,42 @@ mod tests {
|
||||
assert_eq!(ledger.attempted_classes("example").unwrap(), vec![2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_gates_json_still_deserializes() {
|
||||
// Exact shape of the 34 pre-existing rows' gates column: no
|
||||
// tsan_clean / risk_signals / equivalence_mode keys.
|
||||
let old = r#"{"upstream_tests":true,"golden_replay":true,"fuzz_iters":4332,
|
||||
"fuzz_divergence":false,"sanitizers_clean":true}"#;
|
||||
let g: GateResults = serde_json::from_str(old).unwrap();
|
||||
assert!(g.upstream_tests && g.golden_replay && g.sanitizers_clean);
|
||||
assert_eq!(g.fuzz_iters, 4332);
|
||||
assert_eq!(g.tsan_clean, None);
|
||||
assert!(g.risk_signals.is_empty());
|
||||
assert_eq!(g.equivalence_mode, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extended_gates_round_trip() {
|
||||
let g = GateResults {
|
||||
upstream_tests: true,
|
||||
golden_replay: true,
|
||||
fuzz_iters: 10_000,
|
||||
fuzz_divergence: false,
|
||||
sanitizers_clean: true,
|
||||
tsan_clean: Some(false),
|
||||
risk_signals: vec!["concurrency".into()],
|
||||
equivalence_mode: Some("fp-tolerance".into()),
|
||||
};
|
||||
let json = serde_json::to_string(&g).unwrap();
|
||||
let back: GateResults = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back.tsan_clean, Some(false));
|
||||
assert_eq!(back.risk_signals, vec!["concurrency".to_string()]);
|
||||
assert_eq!(back.equivalence_mode.as_deref(), Some("fp-tolerance"));
|
||||
// The keys the defensive Value readers depend on must survive.
|
||||
let v: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(v.get("sanitizers_clean"), Some(&serde_json::json!(true)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_only_is_enforced() {
|
||||
let ledger = Ledger::open_in_memory().unwrap();
|
||||
|
||||
+279
-62
@@ -1,15 +1,25 @@
|
||||
//! `just verdict`: the whole attempt pipeline in one command.
|
||||
//! Gates on the candidate → interleaved A/B bench vs the baseline →
|
||||
//! verdict by the CI-lower-bound rule → append-only ledger row.
|
||||
//! Any gate failure is rejected-gate and the bench is skipped; the
|
||||
//! --needs-human-review flag caps the verdict for FP/concurrency/
|
||||
//! sanitizer-flagged attempts (SPEC §8) — it never upgrades one.
|
||||
//! Any gate failure is rejected-gate and the bench is skipped. A verdict
|
||||
//! is only ever capped, never upgraded: the lexical risk classifier
|
||||
//! (src/risk.rs) and fp-tolerance equivalence route any would-be accept
|
||||
//! to needs-human-review, and the --needs-human-review flag forces the
|
||||
//! same cap by hand (SPEC §8).
|
||||
//!
|
||||
//! This is also the only flow that can run differential fuzz, because it is
|
||||
//! the only one holding both sides of the comparison: it rebuilds a pristine
|
||||
//! baseline and passes it to the gates alongside the candidate. Hard rule —
|
||||
//! a machine `accepted` requires that fuzz gate to have actually Passed; a
|
||||
//! skipped or failed one caps the verdict at needs-human-review.
|
||||
|
||||
mod risk;
|
||||
|
||||
use anyhow::Result;
|
||||
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
||||
use clap::Parser;
|
||||
use diff_test::{
|
||||
all_passed, orchestrate::run_core_gates, target::TargetSpec, GateLayer, GateOutcome,
|
||||
orchestrate::run_core_gates, target::TargetSpec, GateInputs, GateLayer, GateOutcome,
|
||||
};
|
||||
use ledger::{Attempt, BenchEvidence, GateResults, Ledger, Verdict};
|
||||
use std::path::PathBuf;
|
||||
@@ -48,6 +58,10 @@ struct Cli {
|
||||
/// Force the needs-human-review verdict tier (FP/concurrency/UB-adjacent).
|
||||
#[arg(long)]
|
||||
needs_human_review: bool,
|
||||
/// How this run was isolated, recorded verbatim in the env
|
||||
/// fingerprint (e.g. "no-net.sh"); absent = "unwrapped-host".
|
||||
#[arg(long)]
|
||||
isolation_note: Option<String>,
|
||||
#[arg(long, default_value = "results/ledger.sqlite")]
|
||||
db: PathBuf,
|
||||
#[arg(long, default_value = "config/accept.toml")]
|
||||
@@ -107,6 +121,78 @@ fn rebuild_pristine_baseline(
|
||||
Ok(baseline)
|
||||
}
|
||||
|
||||
/// A run is flagged on nonzero exit OR any of these stderr markers.
|
||||
/// halt_on_error makes sanitizer hits exit nonzero; the markers catch
|
||||
/// builds whose runtime still exits 0 (and never a bare "ERROR:" from
|
||||
/// the target's own logging, the old false-flag risk).
|
||||
const SANITIZER_MARKERS: &[&str] = &[
|
||||
"ERROR: AddressSanitizer",
|
||||
"ERROR: LeakSanitizer",
|
||||
"ERROR: ThreadSanitizer",
|
||||
"WARNING: ThreadSanitizer",
|
||||
"runtime error:",
|
||||
];
|
||||
|
||||
/// One instrumented build+run lane: templates, isolation subdir, and the
|
||||
/// halt-on-error runtime options for its sanitizer family.
|
||||
struct SanLane<'a> {
|
||||
label: &'a str,
|
||||
out_subdir: &'a str,
|
||||
build_tpl: &'a str,
|
||||
bin_tpl: &'a str,
|
||||
envs: &'a [(&'a str, &'a str)],
|
||||
}
|
||||
|
||||
/// Build one instrumented lane and run the pinned bench workload under
|
||||
/// it. Returns Some(clean), or None when the instrumented binary cannot
|
||||
/// be built — an infrastructure failure, not a code defect: the caller
|
||||
/// caps the accept at needs-human-review instead of crashing a verdict
|
||||
/// that already carries a measured bench.
|
||||
fn run_san_lane(
|
||||
root: &std::path::Path,
|
||||
target: &str,
|
||||
spec: &TargetSpec,
|
||||
lane: &SanLane,
|
||||
) -> Result<Option<bool>> {
|
||||
let label = lane.label;
|
||||
let out_dir = root
|
||||
.join(format!("targets/{target}/{}", lane.out_subdir))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
println!("{label}: instrumented build of the patched tree…");
|
||||
let status = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(diff_test::target::subst_out(lane.build_tpl, &out_dir))
|
||||
.current_dir(root)
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
println!("{label}: BUILD FAILED — cannot verify; capping accept at needs-human-review");
|
||||
return Ok(None);
|
||||
}
|
||||
let bin = diff_test::target::subst_out(lane.bin_tpl, &out_dir);
|
||||
let mut run = std::process::Command::new("sh");
|
||||
run.arg("-c")
|
||||
.arg(spec.bench.command.replace("{binary}", &bin))
|
||||
.current_dir(root);
|
||||
for (k, v) in lane.envs {
|
||||
run.env(k, v);
|
||||
}
|
||||
let out = run.output()?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let clean = out.status.success() && !SANITIZER_MARKERS.iter().any(|m| stderr.contains(m));
|
||||
if clean {
|
||||
println!("{label}: clean");
|
||||
} else {
|
||||
let lines: Vec<&str> = stderr.lines().filter(|l| !l.is_empty()).collect();
|
||||
let start = lines.len().saturating_sub(8);
|
||||
println!("{label}: FLAGGED (rc={:?})", out.status.code());
|
||||
for l in &lines[start..] {
|
||||
println!(" {l}");
|
||||
}
|
||||
}
|
||||
Ok(Some(clean))
|
||||
}
|
||||
|
||||
/// SPEC §8: an accept is final only if ASan+LSan are clean on the patched
|
||||
/// tree over the pinned workload. Stop-the-line fix after phase2-comrak-010:
|
||||
/// the pipeline auto-accepted an LSan-flagged teardown patch (sanitizers
|
||||
@@ -126,49 +212,67 @@ fn sanitizer_check(
|
||||
println!("sanitizers: target declares no sanitizer build — cannot verify an accept");
|
||||
return Ok(None);
|
||||
};
|
||||
let out_dir = root
|
||||
.join(format!("targets/{target}/asan"))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
println!("sanitizers: ASan build of the patched tree…");
|
||||
let status = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(diff_test::target::subst_out(build_tpl, &out_dir))
|
||||
.current_dir(root)
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
// Infrastructure failure (missing runtime, toolchain), not a code
|
||||
// defect: don't crash the verdict and lose a measured bench —
|
||||
// treat "cannot build the sanitizer binary" like "no sanitizer
|
||||
// declared" so the accept caps at needs-human-review (None), with
|
||||
// the failure logged for the operator to fix and re-run.
|
||||
println!("sanitizers: BUILD FAILED — cannot verify; capping accept at needs-human-review");
|
||||
run_san_lane(
|
||||
root,
|
||||
target,
|
||||
spec,
|
||||
&SanLane {
|
||||
label: "sanitizers",
|
||||
out_subdir: "asan",
|
||||
build_tpl,
|
||||
bin_tpl,
|
||||
envs: &[
|
||||
("ASAN_OPTIONS", "halt_on_error=1:abort_on_error=1"),
|
||||
("UBSAN_OPTIONS", "halt_on_error=1:print_stacktrace=1"),
|
||||
],
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// TSan lane (SPEC §8), run on the accept path when the target declares
|
||||
/// a TSan build — call only when spec.build.tsan is Some. Returns None
|
||||
/// when the lane cannot run (missing tsan_binary, build failure); the
|
||||
/// caller caps the accept, mirroring the ASan posture.
|
||||
fn tsan_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result<Option<bool>> {
|
||||
let (Some(build_tpl), Some(bin_tpl)) = (&spec.build.tsan, &spec.build.tsan_binary) else {
|
||||
println!("tsan: build.tsan declared without build.tsan_binary — cannot verify");
|
||||
return Ok(None);
|
||||
};
|
||||
run_san_lane(
|
||||
root,
|
||||
target,
|
||||
spec,
|
||||
&SanLane {
|
||||
label: "tsan",
|
||||
out_subdir: "tsan",
|
||||
build_tpl,
|
||||
bin_tpl,
|
||||
envs: &[("TSAN_OPTIONS", "halt_on_error=1")],
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// One word for the state of the DifferentialFuzz gate, for the cap message.
|
||||
fn fuzz_state(outcome: Option<&GateOutcome>) -> &'static str {
|
||||
match outcome {
|
||||
Some(GateOutcome::Passed) => "passed",
|
||||
Some(GateOutcome::Failed { .. }) => "failed",
|
||||
Some(GateOutcome::Skipped { .. }) => "skipped",
|
||||
None => "not run",
|
||||
}
|
||||
let asan_bin = diff_test::target::subst_out(bin_tpl, &out_dir);
|
||||
let cmd = spec.bench.command.replace("{binary}", &asan_bin);
|
||||
let out = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(&cmd)
|
||||
// detect_leaks is a no-op for the C/C++ UBSan combo but harmless.
|
||||
.env("ASAN_OPTIONS", "detect_leaks=1")
|
||||
.env("UBSAN_OPTIONS", "halt_on_error=1:print_stacktrace=1")
|
||||
.current_dir(root)
|
||||
.output()?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let clean =
|
||||
out.status.success() && !stderr.contains("ERROR:") && !stderr.contains("runtime error:");
|
||||
if clean {
|
||||
println!("sanitizers: clean");
|
||||
}
|
||||
|
||||
/// Median of per-run max-RSS samples (KiB); an even count averages the
|
||||
/// two middle values.
|
||||
fn median_u64(xs: &[u64]) -> u64 {
|
||||
let mut v = xs.to_vec();
|
||||
v.sort_unstable();
|
||||
let n = v.len();
|
||||
if n % 2 == 1 {
|
||||
v[n / 2]
|
||||
} else {
|
||||
let lines: Vec<&str> = stderr.lines().filter(|l| !l.is_empty()).collect();
|
||||
let start = lines.len().saturating_sub(8);
|
||||
println!("sanitizers: FLAGGED (rc={:?})", out.status.code());
|
||||
for l in &lines[start..] {
|
||||
println!(" {l}");
|
||||
}
|
||||
(v[n / 2 - 1] + v[n / 2]) / 2
|
||||
}
|
||||
Ok(Some(clean))
|
||||
}
|
||||
|
||||
fn now_utc() -> String {
|
||||
@@ -197,23 +301,58 @@ fn main() -> Result<()> {
|
||||
})?
|
||||
};
|
||||
|
||||
// Gates run on the candidate binary.
|
||||
let gates = run_core_gates(&root, &spec, &cli.candidate_bin)?;
|
||||
for (layer, outcome) in &gates {
|
||||
let patch = match &cli.patch_file {
|
||||
Some(p) => std::fs::read_to_string(p)?,
|
||||
None => "(no source patch)".into(),
|
||||
};
|
||||
|
||||
// Gates run on the candidate binary — except differential fuzz, which
|
||||
// differs it against the pristine baseline built just above. This is the
|
||||
// only flow that has both sides, and therefore the only flow that can
|
||||
// run the fuzz gate at all (hence the accept rule enforced below).
|
||||
let report = run_core_gates(
|
||||
&root,
|
||||
&spec,
|
||||
&GateInputs {
|
||||
candidate_binary: &cli.candidate_bin,
|
||||
baseline_binary: Some(baseline_bin.as_str()),
|
||||
fuzz_iters: cfg.fuzz_iters,
|
||||
},
|
||||
)?;
|
||||
for (layer, outcome) in &report.gates {
|
||||
match outcome {
|
||||
GateOutcome::Passed => println!("{layer:?}: PASS"),
|
||||
GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"),
|
||||
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
|
||||
}
|
||||
}
|
||||
let passed = |layer| matches!(report.outcome(layer), Some(GateOutcome::Passed));
|
||||
let mut gate_results = GateResults {
|
||||
upstream_tests: matches!(gates[0].1, GateOutcome::Passed),
|
||||
golden_replay: matches!(gates[1].1, GateOutcome::Passed),
|
||||
fuzz_iters: 0,
|
||||
fuzz_divergence: false,
|
||||
upstream_tests: passed(GateLayer::UpstreamTests),
|
||||
golden_replay: passed(GateLayer::GoldenReplay),
|
||||
fuzz_iters: report.fuzz_iters,
|
||||
fuzz_divergence: report.fuzz_divergence,
|
||||
sanitizers_clean: false,
|
||||
tsan_clean: None,
|
||||
risk_signals: risk::classify(&patch, report.equivalence_mode == "fp-tolerance"),
|
||||
equivalence_mode: Some(report.equivalence_mode.clone()),
|
||||
};
|
||||
debug_assert!(matches!(gates[0].0, GateLayer::UpstreamTests));
|
||||
if !gate_results.risk_signals.is_empty() {
|
||||
println!(
|
||||
"risk: lexical signals [{}] — a would-be accept routes to needs-human-review (SPEC §8)",
|
||||
gate_results.risk_signals.join(", ")
|
||||
);
|
||||
}
|
||||
// Announced up front, like the risk signals, so the rule is in the
|
||||
// transcript even when another cap fires first below.
|
||||
let fuzz_passed = passed(GateLayer::DifferentialFuzz);
|
||||
if !fuzz_passed {
|
||||
println!(
|
||||
"fuzz: differential-fuzz gate {} — a would-be accept routes to needs-human-review; \
|
||||
no machine accept without a real differential-fuzz run (SPEC §8)",
|
||||
fuzz_state(report.outcome(GateLayer::DifferentialFuzz))
|
||||
);
|
||||
}
|
||||
|
||||
let pin = &cfg.pin_prefix;
|
||||
let wrap = |bin: &str| {
|
||||
@@ -225,7 +364,7 @@ fn main() -> Result<()> {
|
||||
}
|
||||
};
|
||||
|
||||
let (verdict, bench) = if !all_passed(&gates) {
|
||||
let (verdict, bench) = if !report.all_passed() {
|
||||
println!("verdict: rejected-gate (bench skipped)");
|
||||
(Verdict::RejectedGate, None)
|
||||
} else {
|
||||
@@ -285,8 +424,62 @@ fn main() -> Result<()> {
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
}
|
||||
if spec.build.tsan.is_some() {
|
||||
match tsan_check(&root, &cli.target, &spec)? {
|
||||
Some(clean) => {
|
||||
gate_results.tsan_clean = Some(clean);
|
||||
if !clean {
|
||||
println!(
|
||||
"verdict: TSan flag caps the accept at needs-human-review (SPEC §8)"
|
||||
);
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
println!("verdict: TSan lane declared but not verifiable; capping accept at needs-human-review (SPEC §8)");
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
}
|
||||
}
|
||||
if v == Verdict::Accepted && !gate_results.risk_signals.is_empty() {
|
||||
println!(
|
||||
"verdict: risk signals [{}] route the accept to needs-human-review (SPEC §8)",
|
||||
gate_results.risk_signals.join(", ")
|
||||
);
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
// Hard rule (audit remediation): differential fuzz is the gate
|
||||
// that catches behavioral divergence the fixed corpus misses, so
|
||||
// an accept minted without it is an unverified claim. The audit
|
||||
// found accepted rows carrying fuzz_iters=0 — fuzz had run
|
||||
// out-of-band, or not at all. A skipped or failed fuzz gate can
|
||||
// no longer produce a machine accept, only needs-human-review.
|
||||
if v == Verdict::Accepted && !fuzz_passed {
|
||||
println!(
|
||||
"verdict: differential fuzz {} — no accept without a real differential-fuzz \
|
||||
run against the pristine baseline; capping at needs-human-review. The target \
|
||||
must declare [gates].fuzz in targets/{}/target.toml, and the run must report \
|
||||
FUZZ-RESULT with zero divergences (SPEC §8)",
|
||||
fuzz_state(report.outcome(GateLayer::DifferentialFuzz)),
|
||||
cli.target
|
||||
);
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
}
|
||||
let fp = EnvFingerprint::collect(pin, "system-default");
|
||||
let mut env_fingerprint = serde_json::json!({
|
||||
"fingerprint": fp,
|
||||
"workload": spec.bench.workload,
|
||||
"target_commit": spec.source.commit,
|
||||
"isolation": cli.isolation_note.as_deref().unwrap_or("unwrapped-host"),
|
||||
"gates_detail": "diff-fuzz gated in-pipeline against the pristine baseline (an accept requires it to have passed); ASan+UBSan enforced on the accept path, TSan when configured",
|
||||
});
|
||||
if !samples.baseline_max_rss_kib.is_empty() && !samples.candidate_max_rss_kib.is_empty() {
|
||||
env_fingerprint["max_rss_kib"] = serde_json::json!({
|
||||
"baseline_median": median_u64(&samples.baseline_max_rss_kib),
|
||||
"candidate_median": median_u64(&samples.candidate_max_rss_kib),
|
||||
});
|
||||
}
|
||||
(
|
||||
v,
|
||||
Some(BenchEvidence {
|
||||
@@ -296,20 +489,11 @@ fn main() -> Result<()> {
|
||||
candidate_ci: (clo, chi),
|
||||
speedup_median: ratio.median,
|
||||
speedup_ci: (ratio.lo, ratio.hi),
|
||||
env_fingerprint: serde_json::json!({
|
||||
"fingerprint": fp,
|
||||
"workload": spec.bench.workload,
|
||||
"target_commit": spec.source.commit,
|
||||
"gates_detail": "diff-fuzz per-attempt manual; ASan+LSan enforced on the accept path",
|
||||
}),
|
||||
env_fingerprint,
|
||||
}),
|
||||
)
|
||||
};
|
||||
|
||||
let patch = match &cli.patch_file {
|
||||
Some(p) => std::fs::read_to_string(p)?,
|
||||
None => "(no source patch)".into(),
|
||||
};
|
||||
// Derive the phase from the run_id namespace (phaseN-*) so the column
|
||||
// matches the run_id prefix; the historical hardcoded `1` left every
|
||||
// verdict-written row tagged phase 1 regardless of namespace (found in
|
||||
@@ -344,3 +528,36 @@ fn main() -> Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{fuzz_state, median_u64};
|
||||
use diff_test::GateOutcome;
|
||||
|
||||
#[test]
|
||||
fn median_u64_odd_and_even() {
|
||||
assert_eq!(median_u64(&[5]), 5);
|
||||
assert_eq!(median_u64(&[9, 1, 5]), 5);
|
||||
assert_eq!(median_u64(&[4, 2, 8, 6]), 5);
|
||||
}
|
||||
|
||||
/// Only `Passed` clears the accept bar; every other state (including a
|
||||
/// gate that never ran) is a cap to needs-human-review.
|
||||
#[test]
|
||||
fn fuzz_state_names_every_outcome() {
|
||||
assert_eq!(fuzz_state(Some(&GateOutcome::Passed)), "passed");
|
||||
assert_eq!(
|
||||
fuzz_state(Some(&GateOutcome::Failed {
|
||||
detail: "divergences=3".into()
|
||||
})),
|
||||
"failed"
|
||||
);
|
||||
assert_eq!(
|
||||
fuzz_state(Some(&GateOutcome::Skipped {
|
||||
reason: "no baseline".into()
|
||||
})),
|
||||
"skipped"
|
||||
);
|
||||
assert_eq!(fuzz_state(None), "not run");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
//! Lexical NHR risk classifier (SPEC §8). Scans only the changed lines
|
||||
//! of a unified diff for concurrency / unsafe / floating-point tokens
|
||||
//! and routes any would-be accept that hits one to needs-human-review.
|
||||
//! Deliberately lexical and conservative: over-triggering costs a human
|
||||
//! a look, under-triggering ships an unreviewed risk — so plain token
|
||||
//! presence decides, no semantic analysis.
|
||||
|
||||
const CONCURRENCY: &[&str] = &[
|
||||
"pthread_",
|
||||
"std::sync",
|
||||
"std::thread",
|
||||
"Atomic",
|
||||
"atomic_",
|
||||
"__atomic_",
|
||||
"__sync_",
|
||||
"Mutex",
|
||||
"RwLock",
|
||||
"Condvar",
|
||||
"mpsc",
|
||||
"rayon::",
|
||||
"memory_order",
|
||||
"volatile",
|
||||
];
|
||||
|
||||
const UNSAFE: &[&str] = &[
|
||||
"unsafe ",
|
||||
"transmute",
|
||||
"from_raw",
|
||||
"MaybeUninit",
|
||||
"UnsafeCell",
|
||||
];
|
||||
|
||||
/// The spaced tokens carry their own trailing boundary; `f32`/`f64` get
|
||||
/// hand boundary checks below.
|
||||
const FP_SPACED: &[&str] = &["float ", "double "];
|
||||
const FP_TYPES: &[&str] = &["f32", "f64"];
|
||||
|
||||
fn is_ident_letter(b: u8) -> bool {
|
||||
b.is_ascii_alphabetic() || b == b'_'
|
||||
}
|
||||
|
||||
/// True when `line` contains `token` not glued to an identifier letter on
|
||||
/// either side. Digits are allowed neighbors so literal suffixes like
|
||||
/// `1.0f32` still trigger; letter neighbors (identifiers, most hex
|
||||
/// strings, e.g. `buf32`, `deadf64b`) do not.
|
||||
fn has_fp_token(line: &str, token: &str) -> bool {
|
||||
let bytes = line.as_bytes();
|
||||
let mut start = 0;
|
||||
while let Some(pos) = line[start..].find(token) {
|
||||
let i = start + pos;
|
||||
let before_ok = i == 0 || !is_ident_letter(bytes[i - 1]);
|
||||
let after_ok = i + token.len() >= bytes.len() || !is_ident_letter(bytes[i + token.len()]);
|
||||
if before_ok && after_ok {
|
||||
return true;
|
||||
}
|
||||
start = i + 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_changed_line(line: &str) -> bool {
|
||||
(line.starts_with('+') && !line.starts_with("+++"))
|
||||
|| (line.starts_with('-') && !line.starts_with("---"))
|
||||
}
|
||||
|
||||
/// Classify a unified diff into risk-signal names. `fp_tolerance` is true
|
||||
/// when the target's equivalence mode is fp-tolerance, itself a §8 signal.
|
||||
/// Returned names are unique; empty means no signal.
|
||||
pub fn classify(diff: &str, fp_tolerance: bool) -> Vec<String> {
|
||||
let (mut concurrency, mut unsafe_sig, mut fp) = (false, false, false);
|
||||
for line in diff.lines().filter(|l| is_changed_line(l)) {
|
||||
concurrency = concurrency || CONCURRENCY.iter().any(|t| line.contains(t));
|
||||
unsafe_sig = unsafe_sig || UNSAFE.iter().any(|t| line.contains(t));
|
||||
fp = fp
|
||||
|| FP_SPACED.iter().any(|t| line.contains(t))
|
||||
|| FP_TYPES.iter().any(|t| has_fp_token(line, t));
|
||||
}
|
||||
let mut signals = Vec::new();
|
||||
if concurrency {
|
||||
signals.push("concurrency".to_string());
|
||||
}
|
||||
if unsafe_sig {
|
||||
signals.push("unsafe".to_string());
|
||||
}
|
||||
if fp {
|
||||
signals.push("floating-point".to_string());
|
||||
}
|
||||
if fp_tolerance {
|
||||
signals.push("fp-tolerance-equivalence".to_string());
|
||||
}
|
||||
signals
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn concurrency_tokens_trigger() {
|
||||
let diff = "--- a/src/lib.rs\n+++ b/src/lib.rs\n+ let m = Mutex::new(0);\n";
|
||||
assert_eq!(classify(diff, false), vec!["concurrency"]);
|
||||
let diff = "- pthread_create(&t, NULL, worker, NULL);\n";
|
||||
assert_eq!(classify(diff, false), vec!["concurrency"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsafe_tokens_trigger() {
|
||||
let diff = "+ unsafe { ptr.read() }\n";
|
||||
assert_eq!(classify(diff, false), vec!["unsafe"]);
|
||||
let diff = "+ let v: Vec<u8> = Vec::from_raw_parts(p, n, n);\n";
|
||||
assert_eq!(classify(diff, false), vec!["unsafe"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floating_point_tokens_trigger_with_boundaries() {
|
||||
assert_eq!(
|
||||
classify("+ let x: f64 = 1.0;\n", false),
|
||||
vec!["floating-point"]
|
||||
);
|
||||
// Literal suffixes have a digit neighbor and must still trigger.
|
||||
assert_eq!(
|
||||
classify("+ sum += 1.0f32;\n", false),
|
||||
vec!["floating-point"]
|
||||
);
|
||||
assert_eq!(
|
||||
classify("+ double acc = 0.0;\n", false),
|
||||
vec!["floating-point"]
|
||||
);
|
||||
// Glued to identifier letters: not an FP token.
|
||||
assert!(classify("+ let buf32 = [0u8; 4];\n", false).is_empty());
|
||||
assert!(classify("+ hash = \"deadf64beef\";\n", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fp_tolerance_flag_is_its_own_signal() {
|
||||
assert_eq!(classify("", true), vec!["fp-tolerance-equivalence"]);
|
||||
assert_eq!(
|
||||
classify("+ let x: f32 = 0.5;\n", true),
|
||||
vec!["floating-point", "fp-tolerance-equivalence"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_diff_yields_no_signals() {
|
||||
let diff = "--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -1,3 +1,3 @@\n\
|
||||
- let n = items.len();\n+ let n = count;\n let done = true;\n";
|
||||
assert!(classify(diff, false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_changed_lines_are_scanned() {
|
||||
// Tokens on context lines and in file headers must not trigger.
|
||||
let diff = "--- a/src/mutex_wrapper.rs\n+++ b/src/mutex_wrapper.rs\n\
|
||||
@@ -10,4 +10,4 @@\n let guard = lock.mutex();\n\
|
||||
+ let n = count;\n";
|
||||
assert!(classify(diff, false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signals_are_deduped() {
|
||||
let diff = "+ let a = Mutex::new(0);\n+ let b = RwLock::new(0);\n\
|
||||
+ unsafe { transmute::<u32, f32>(x) };\n";
|
||||
assert_eq!(
|
||||
classify(diff, false),
|
||||
vec!["concurrency", "unsafe", "floating-point"]
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user