mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
gates: close the audit findings — enforce the gates the docs claimed
An adversarial audit found the enforcement story ahead of the code: gates that were documented as hard gates were advisory, manual, or absent, and several human-facing claims outran the machine record. This makes the claims true. High findings - Differential fuzz was documentation, not enforcement: verdict hardcoded fuzz_iters=0 and diff-test hardcoded the gate to Skipped, while the case studies claimed 10k-iteration fuzz on the same wins. Fuzz is now a real pipeline gate (strict FUZZ-RESULT contract, executed count recorded), and it is inherently baseline-vs-candidate, so it runs on the accept path and reports Skipped in the standalone `just gates` flow. New hard rule: a machine `accepted` verdict is impossible unless the fuzz gate actually passed — otherwise it caps at needs-human-review. - The agent's system prompt claimed FP/concurrency/unsafe patches were "routed to human review automatically". No such mechanism existed. Added a conservative lexical risk classifier (crates/verdict/src/risk.rs) that forces needs-human-review on any would-be accept carrying those signals; fp-tolerance equivalence always signals. - The local verdict path ran agent-influenced code unsandboxed. It now runs under scripts/no-net.sh (network namespace) and fails closed. Upstream test suites are hash-verified against corpora/<t>/TESTSUITE.sha256 when present (scripts/pin-testsuite.sh generates them). Also fixed, found while fixing the above - The pipeline script's `set -e` was inert (brace group as the left operand of `&&`), so a failing candidate build did not abort the run and verdict could bench a stale binary. Now a scoped subshell whose status is branched on at top level. - A wrapper that exits before exec'ing (no-net.sh refusing, bad wrapper path) stranded RUNNING.lock and left read_verdict polling forever. Failure accounting now lives outside the wrapper. - harnessd's patch allowlist ignored git rename/copy headers and --no-prefix diffs; the rename-escape case is now a committed regression test. - The workspace did not compile on Windows (ungated Unix APIs). cfg-gated; 71 portable tests now run on both platforms. - pin-check regenerated the corpus manifest instead of verifying it; it is now verify-only, with a deliberate `just pin-corpus` to re-pin. - Max RSS is captured per run on Unix; PMU counters are opt-in diagnostics (compare --perf-stat), never part of the accept decision. Docs now match the code, including what is NOT yet enforced: no target ships a TESTSUITE.sha256 pin, no target declares [build].tsan, and matmul is not wired to the fp-tolerance gate — each is stated as mechanized-but-inert rather than done. Corrected the stale "33 ledger rows" (34), the playbook's class-3 cJSON win (rejected), adversarial-review.md's sanitizer-separation claim (five accepted rows carry sanitizers_clean=false), the unsupported "40,000 requests" figure, and the blanket "historical rows record fuzz_iters=0" (8 of 10; phase0-tokei-002/-003 record 4,332). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -43,6 +43,49 @@ pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool {
|
||||
.any(|(_, o)| matches!(o, GateOutcome::Passed))
|
||||
}
|
||||
|
||||
/// Inputs to one gate run: the candidate binary under test, the pristine
|
||||
/// baseline to differ it against, and the differential-fuzz iteration
|
||||
/// budget (from AcceptConfig, default 10_000).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct GateInputs<'a> {
|
||||
/// Patched binary (repo-root-relative): golden replay runs it, and it
|
||||
/// is the `{candidate}` side of differential fuzz.
|
||||
pub candidate_binary: &'a str,
|
||||
/// Pristine baseline binary (repo-root-relative), the `{baseline}` side
|
||||
/// of differential fuzz. `None` when the caller has no baseline to
|
||||
/// differ against — the standalone `just gates` flow builds only the
|
||||
/// working tree — and the fuzz gate is then skipped with that reason
|
||||
/// rather than silently comparing the candidate to itself. The accept
|
||||
/// path (`just verdict`) rebuilds a pristine baseline and passes it,
|
||||
/// and only a fuzz gate that actually Passed can mint an accept.
|
||||
pub baseline_binary: Option<&'a str>,
|
||||
pub fuzz_iters: u64,
|
||||
}
|
||||
|
||||
/// Everything one gate run produced: the four layers in fixed order
|
||||
/// (UpstreamTests, GoldenReplay, DifferentialFuzz, Sanitizers) plus the
|
||||
/// fuzz counts and equivalence mode that verdict records in the ledger.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct GateReport {
|
||||
pub gates: Vec<(GateLayer, GateOutcome)>,
|
||||
/// Fuzz iterations actually executed (parsed from the FUZZ-RESULT
|
||||
/// line); 0 when the fuzz gate was skipped or never reported.
|
||||
pub fuzz_iters: u64,
|
||||
pub fuzz_divergence: bool,
|
||||
/// "byte-identical" | "fp-tolerance"
|
||||
pub equivalence_mode: String,
|
||||
}
|
||||
|
||||
impl GateReport {
|
||||
pub fn outcome(&self, layer: GateLayer) -> Option<&GateOutcome> {
|
||||
self.gates.iter().find(|(l, _)| *l == layer).map(|(_, o)| o)
|
||||
}
|
||||
|
||||
pub fn all_passed(&self) -> bool {
|
||||
all_passed(&self.gates)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -71,4 +114,32 @@ mod tests {
|
||||
)];
|
||||
assert!(!all_passed(&results));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_outcome_looks_up_by_layer() {
|
||||
let report = GateReport {
|
||||
gates: vec![
|
||||
(GateLayer::UpstreamTests, GateOutcome::Passed),
|
||||
(
|
||||
GateLayer::GoldenReplay,
|
||||
GateOutcome::Failed {
|
||||
detail: "stdout diverged".into(),
|
||||
},
|
||||
),
|
||||
],
|
||||
fuzz_iters: 0,
|
||||
fuzz_divergence: false,
|
||||
equivalence_mode: "byte-identical".into(),
|
||||
};
|
||||
assert_eq!(
|
||||
report.outcome(GateLayer::UpstreamTests),
|
||||
Some(&GateOutcome::Passed)
|
||||
);
|
||||
assert!(matches!(
|
||||
report.outcome(GateLayer::GoldenReplay),
|
||||
Some(GateOutcome::Failed { .. })
|
||||
));
|
||||
assert_eq!(report.outcome(GateLayer::Sanitizers), None);
|
||||
assert!(!report.all_passed());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
//! Gate orchestration CLI: corpus pin → upstream tests → golden replay.
|
||||
//! Differential fuzz and sanitizers remain per-attempt manual steps in
|
||||
//! Phase 1 and are reported as skipped with a reason.
|
||||
//! Gate orchestration CLI: corpus + test-suite pins → upstream tests →
|
||||
//! policy-aware golden replay → differential fuzz. Sanitizers run on the
|
||||
//! accept path in verdict and are reported as skipped with a reason.
|
||||
//!
|
||||
//! Differential fuzz is skipped here too, and for the same structural
|
||||
//! reason: it differs a *pristine baseline* against the candidate, and this
|
||||
//! flow builds only the working tree — there is nothing to differ against.
|
||||
//! The accept path (`just verdict`) rebuilds a pristine baseline, passes it
|
||||
//! in, and refuses to mint an accept unless the fuzz gate actually passed.
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use diff_test::orchestrate::run_core_gates;
|
||||
use diff_test::target::{subst_out, TargetSpec};
|
||||
use diff_test::{all_passed, GateOutcome};
|
||||
use diff_test::{GateInputs, GateLayer, GateOutcome};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
@@ -21,6 +27,13 @@ struct Cli {
|
||||
/// tree is built to targets/<name>/gates-build and that is tested.
|
||||
#[arg(long)]
|
||||
binary: Option<String>,
|
||||
/// Differential-fuzz iteration budget, substituted for {iters} in the
|
||||
/// target's [gates].fuzz command. Inert on this flow: with no pristine
|
||||
/// baseline to differ against, the fuzz gate is skipped. The accept path
|
||||
/// (`just verdict`) rebuilds one and takes its budget from
|
||||
/// config/accept.toml.
|
||||
#[arg(long, default_value_t = 10_000)]
|
||||
fuzz_iters: u64,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
@@ -47,15 +60,42 @@ fn main() -> Result<()> {
|
||||
subst_out(&spec.build.binary, &out)
|
||||
}
|
||||
};
|
||||
let results = run_core_gates(&root, &spec, &binary)?;
|
||||
for (layer, outcome) in &results {
|
||||
let report = run_core_gates(
|
||||
&root,
|
||||
&spec,
|
||||
&GateInputs {
|
||||
candidate_binary: &binary,
|
||||
// No pristine baseline on this flow — see the module doc. The
|
||||
// fuzz gate reports Skipped with that reason rather than
|
||||
// comparing the candidate against itself.
|
||||
baseline_binary: None,
|
||||
fuzz_iters: cli.fuzz_iters,
|
||||
},
|
||||
)?;
|
||||
for (layer, outcome) in &report.gates {
|
||||
match outcome {
|
||||
GateOutcome::Passed if *layer == GateLayer::DifferentialFuzz => {
|
||||
println!("{layer:?}: PASS ({} iters)", report.fuzz_iters)
|
||||
}
|
||||
GateOutcome::Passed => println!("{layer:?}: PASS"),
|
||||
GateOutcome::Failed { detail } => println!("{layer:?}: FAIL — {detail}"),
|
||||
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
|
||||
}
|
||||
}
|
||||
if all_passed(&results) {
|
||||
if report.all_passed() {
|
||||
// Inside all_passed(), the fuzz gate is Passed or Skipped — never
|
||||
// Failed. Say so plainly: a PASS here must not read as a verified
|
||||
// differential-fuzz run, which is exactly the confusion that let
|
||||
// accepted rows carry fuzz_iters=0.
|
||||
if !matches!(
|
||||
report.outcome(GateLayer::DifferentialFuzz),
|
||||
Some(GateOutcome::Passed)
|
||||
) {
|
||||
println!(
|
||||
"note: differential fuzz did not run — `just gates` is not an accept path; only \
|
||||
`just verdict` (pristine baseline + fuzz gate) can accept a change"
|
||||
);
|
||||
}
|
||||
println!("gates: PASS ({})", cli.target);
|
||||
Ok(())
|
||||
} else {
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
//! Core gate sequence, callable from the diff-test CLI and from verdict:
|
||||
//! corpus pin (refuses on mismatch) → upstream tests → golden replay.
|
||||
//! corpus + test-suite pins (refuse on mismatch) → upstream tests →
|
||||
//! policy-aware golden replay → differential fuzz. Sanitizers run on the
|
||||
//! accept path in verdict and are reported here as skipped with a reason.
|
||||
//!
|
||||
//! Differential fuzz is the one gate that is not purely a property of the
|
||||
//! candidate: it differs the candidate against a *pristine baseline*, so it
|
||||
//! needs both binaries. Callers that have only a candidate (the standalone
|
||||
//! `just gates` flow) pass `baseline_binary: None` and the gate is skipped
|
||||
//! with that reason recorded — it is never faked by comparing the candidate
|
||||
//! against itself, and verdict refuses to mint an accept without it.
|
||||
|
||||
use crate::policy::EquivalencePolicy;
|
||||
use crate::target::{expected_golden_hash, TargetSpec};
|
||||
use crate::{pin, GateLayer, GateOutcome};
|
||||
use crate::{pin, GateInputs, GateLayer, GateOutcome, GateReport};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
fn sh(root: &Path, cmd: &str) -> anyhow::Result<std::process::Output> {
|
||||
@@ -17,14 +27,155 @@ fn sh(root: &Path, cmd: &str) -> anyhow::Result<std::process::Output> {
|
||||
.output()?)
|
||||
}
|
||||
|
||||
/// Run the mechanical gates for `target`, testing `binary` (repo-root-
|
||||
/// relative) in the golden replay. Corpus-pin mismatch is an error, not
|
||||
/// a gate failure: nothing runs on a tampered corpus.
|
||||
/// TESTSUITE.sha256 lives beside the corpus MANIFEST.sha256 (both under
|
||||
/// corpora/<target>/), so its path derives from the manifest's.
|
||||
fn suite_manifest_path(corpus_manifest: &Path) -> PathBuf {
|
||||
corpus_manifest
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new(""))
|
||||
.join("TESTSUITE.sha256")
|
||||
}
|
||||
|
||||
fn parse_u64_strict(s: &str) -> Option<u64> {
|
||||
if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
s.parse().ok()
|
||||
}
|
||||
|
||||
/// Parse the last `FUZZ-RESULT iters=<u64> divergences=<u64>` stdout line
|
||||
/// of a fuzz command (exact ASCII form, single spaces). Returns
|
||||
/// `(iters, divergences)`, or None when no line matches.
|
||||
pub fn parse_fuzz_result(stdout: &str) -> Option<(u64, u64)> {
|
||||
stdout.lines().rev().find_map(|line| {
|
||||
let line = line.strip_suffix('\r').unwrap_or(line);
|
||||
let rest = line.strip_prefix("FUZZ-RESULT iters=")?;
|
||||
let (iters, rest) = rest.split_once(' ')?;
|
||||
let divergences = rest.strip_prefix("divergences=")?;
|
||||
Some((parse_u64_strict(iters)?, parse_u64_strict(divergences)?))
|
||||
})
|
||||
}
|
||||
|
||||
/// Recorded verbatim when the target declares no fuzz command at all.
|
||||
const FUZZ_SKIP_NO_COMMAND: &str = "no fuzz command declared in target.toml [gates].fuzz";
|
||||
|
||||
/// Recorded verbatim when a fuzz command exists but the caller has no
|
||||
/// pristine baseline: differential fuzz differs two binaries, and comparing
|
||||
/// the candidate against itself would be a gate that cannot fail.
|
||||
const FUZZ_SKIP_NO_BASELINE: &str =
|
||||
"differential fuzz needs a pristine baseline binary; run the accept path (`just verdict`), \
|
||||
which rebuilds one";
|
||||
|
||||
/// Substitute the three `[gates].fuzz` template placeholders. The contract
|
||||
/// with `targets/<t>/target.toml` and `scripts/diff-fuzz-*.py`: `{iters}`
|
||||
/// is the iteration budget, `{baseline}` the pristine baseline binary and
|
||||
/// `{candidate}` the patched one (both repo-root-relative paths, and the
|
||||
/// command runs from the repo root). A fuzz script must never hardcode a
|
||||
/// baseline path — the harness owns where the pristine build lands.
|
||||
fn fuzz_command(tmpl: &str, iters: u64, baseline: &str, candidate: &str) -> String {
|
||||
tmpl.replace("{iters}", &iters.to_string())
|
||||
.replace("{baseline}", baseline)
|
||||
.replace("{candidate}", candidate)
|
||||
}
|
||||
|
||||
/// What the differential-fuzz gate will do, decided before any process runs.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum FuzzPlan {
|
||||
Skip(&'static str),
|
||||
Run(String),
|
||||
}
|
||||
|
||||
/// Differential fuzz is a baseline-vs-candidate gate: it needs both sides.
|
||||
/// Missing either one is a Skip with the reason recorded, never a silent
|
||||
/// self-comparison and never a hard error (the standalone gates flow has
|
||||
/// no baseline by construction).
|
||||
fn plan_fuzz(fuzz: Option<&str>, inputs: &GateInputs) -> FuzzPlan {
|
||||
match (fuzz, inputs.baseline_binary) {
|
||||
(None, _) => FuzzPlan::Skip(FUZZ_SKIP_NO_COMMAND),
|
||||
(Some(_), None) => FuzzPlan::Skip(FUZZ_SKIP_NO_BASELINE),
|
||||
(Some(tmpl), Some(baseline)) => FuzzPlan::Run(fuzz_command(
|
||||
tmpl,
|
||||
inputs.fuzz_iters,
|
||||
baseline,
|
||||
inputs.candidate_binary,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Grade one executed fuzz run. Passed iff the command exited 0, printed a
|
||||
/// well-formed `FUZZ-RESULT` line, and reported zero divergences — a run
|
||||
/// that never reports the line is a Failed gate, not a pass by silence.
|
||||
/// Returns `(outcome, iters_actually_executed, divergence)`.
|
||||
fn fuzz_gate_outcome(
|
||||
stdout: &str,
|
||||
success: bool,
|
||||
exit_code: Option<i32>,
|
||||
) -> (GateOutcome, u64, bool) {
|
||||
match parse_fuzz_result(stdout) {
|
||||
None => (
|
||||
GateOutcome::Failed {
|
||||
detail: "fuzz command did not report FUZZ-RESULT".into(),
|
||||
},
|
||||
0,
|
||||
false,
|
||||
),
|
||||
Some((iters, divergences)) => (
|
||||
if success && divergences == 0 {
|
||||
GateOutcome::Passed
|
||||
} else {
|
||||
GateOutcome::Failed {
|
||||
detail: format!("exit={exit_code:?} iters={iters} divergences={divergences}"),
|
||||
}
|
||||
},
|
||||
iters,
|
||||
divergences > 0,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// The fp-tolerance policy fails closed without a committed reference:
|
||||
/// there is no hash to fall back to that a tolerance could compare against.
|
||||
fn fp_reference(golden_reference: Option<&PathBuf>) -> Result<&PathBuf, GateOutcome> {
|
||||
golden_reference.ok_or_else(|| GateOutcome::Failed {
|
||||
detail: "fp-tolerance policy requires corpus.golden_reference in target.toml".into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// GoldenReplay verdict under an fp-tolerance policy: the replay stdout is
|
||||
/// compared token-wise against the committed reference output.
|
||||
fn fp_golden_outcome(
|
||||
policy: &EquivalencePolicy,
|
||||
reference: &str,
|
||||
stdout: &str,
|
||||
success: bool,
|
||||
exit_code: Option<i32>,
|
||||
) -> GateOutcome {
|
||||
if !success {
|
||||
return GateOutcome::Failed {
|
||||
detail: format!("golden replay command failed (exit={exit_code:?})"),
|
||||
};
|
||||
}
|
||||
match policy.compare(reference, stdout) {
|
||||
Ok(()) => GateOutcome::Passed,
|
||||
Err(d) => GateOutcome::Failed {
|
||||
detail: format!(
|
||||
"diverged at token/offset {}: baseline={:?} candidate={:?} — {}",
|
||||
d.token, d.baseline, d.candidate, d.reason
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the mechanical gates for `target`, testing `inputs.candidate_binary`
|
||||
/// (repo-root-relative) in golden replay and differential fuzz. Differential
|
||||
/// fuzz additionally needs `inputs.baseline_binary` to differ against and is
|
||||
/// skipped-with-reason without one. Pin mismatches (corpus or test suite)
|
||||
/// are errors, not gate failures: nothing runs on tampered inputs.
|
||||
pub fn run_core_gates(
|
||||
root: &Path,
|
||||
spec: &TargetSpec,
|
||||
binary: &str,
|
||||
) -> anyhow::Result<Vec<(GateLayer, GateOutcome)>> {
|
||||
inputs: &GateInputs,
|
||||
) -> anyhow::Result<GateReport> {
|
||||
let verified = pin::verify_manifest(
|
||||
&root.join(&spec.corpus.manifest),
|
||||
&root.join(&spec.corpus.root),
|
||||
@@ -34,6 +185,24 @@ pub fn run_core_gates(
|
||||
spec.corpus.manifest.display()
|
||||
);
|
||||
|
||||
// Test-suite pin: same refuse-on-mismatch posture as the corpus pin,
|
||||
// but the manifest is optional until a human deliberately generates it.
|
||||
let suite_manifest = suite_manifest_path(&spec.corpus.manifest);
|
||||
if root.join(&suite_manifest).exists() {
|
||||
let verified = pin::verify_manifest(&root.join(&suite_manifest), root).map_err(|e| {
|
||||
anyhow::anyhow!("upstream test suite hash mismatch — refusing to run: {e}")
|
||||
})?;
|
||||
eprintln!(
|
||||
"test-suite pin: {verified} files verified against {}",
|
||||
suite_manifest.display()
|
||||
);
|
||||
} else {
|
||||
eprintln!(
|
||||
"test-suite pin: {} not found — suite unpinned (generate with scripts/pin-testsuite.sh)",
|
||||
suite_manifest.display()
|
||||
);
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
let t = sh(root, &spec.gates.tests)?;
|
||||
@@ -51,34 +220,285 @@ pub fn run_core_gates(
|
||||
},
|
||||
));
|
||||
|
||||
let g = sh(root, &spec.gates.golden.replace("{binary}", binary))?;
|
||||
let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
|
||||
let actual = format!("{:x}", Sha256::digest(&g.stdout));
|
||||
results.push((
|
||||
GateLayer::GoldenReplay,
|
||||
if g.status.success() && actual == expected {
|
||||
GateOutcome::Passed
|
||||
} else {
|
||||
GateOutcome::Failed {
|
||||
detail: format!(
|
||||
"exit={:?} expected={expected} actual={actual}",
|
||||
g.status.code()
|
||||
),
|
||||
let policy = EquivalencePolicy::load(&spec.target_dir(root))?;
|
||||
let equivalence_mode = match &policy {
|
||||
EquivalencePolicy::ByteIdentical => "byte-identical",
|
||||
EquivalencePolicy::FpTolerance { .. } => "fp-tolerance",
|
||||
};
|
||||
let golden_cmd = spec
|
||||
.gates
|
||||
.golden
|
||||
.replace("{binary}", inputs.candidate_binary);
|
||||
let golden = match &policy {
|
||||
EquivalencePolicy::ByteIdentical => {
|
||||
let g = sh(root, &golden_cmd)?;
|
||||
let expected = expected_golden_hash(&root.join(&spec.corpus.golden_sha256))?;
|
||||
let actual = format!("{:x}", Sha256::digest(&g.stdout));
|
||||
if g.status.success() && actual == expected {
|
||||
GateOutcome::Passed
|
||||
} else {
|
||||
GateOutcome::Failed {
|
||||
detail: format!(
|
||||
"exit={:?} expected={expected} actual={actual}",
|
||||
g.status.code()
|
||||
),
|
||||
}
|
||||
}
|
||||
},
|
||||
));
|
||||
}
|
||||
EquivalencePolicy::FpTolerance { .. } => {
|
||||
match fp_reference(spec.corpus.golden_reference.as_ref()) {
|
||||
Err(outcome) => outcome,
|
||||
Ok(reference_path) => {
|
||||
let reference =
|
||||
std::fs::read_to_string(root.join(reference_path)).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"cannot read corpus.golden_reference {}: {e}",
|
||||
reference_path.display()
|
||||
)
|
||||
})?;
|
||||
let g = sh(root, &golden_cmd)?;
|
||||
fp_golden_outcome(
|
||||
&policy,
|
||||
&reference,
|
||||
&String::from_utf8_lossy(&g.stdout),
|
||||
g.status.success(),
|
||||
g.status.code(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
results.push((GateLayer::GoldenReplay, golden));
|
||||
|
||||
let (fuzz_outcome, fuzz_iters, fuzz_divergence) =
|
||||
match plan_fuzz(spec.gates.fuzz.as_deref(), inputs) {
|
||||
FuzzPlan::Skip(reason) => (
|
||||
GateOutcome::Skipped {
|
||||
reason: reason.to_string(),
|
||||
},
|
||||
0,
|
||||
false,
|
||||
),
|
||||
FuzzPlan::Run(cmd) => {
|
||||
let f = sh(root, &cmd)?;
|
||||
fuzz_gate_outcome(
|
||||
&String::from_utf8_lossy(&f.stdout),
|
||||
f.status.success(),
|
||||
f.status.code(),
|
||||
)
|
||||
}
|
||||
};
|
||||
results.push((GateLayer::DifferentialFuzz, fuzz_outcome));
|
||||
|
||||
results.push((
|
||||
GateLayer::DifferentialFuzz,
|
||||
GateOutcome::Skipped {
|
||||
reason: "per-attempt manual step in Phase 1 (needs old/new pair)".into(),
|
||||
},
|
||||
));
|
||||
results.push((
|
||||
GateLayer::Sanitizers,
|
||||
GateOutcome::Skipped {
|
||||
reason: "per-attempt manual step in Phase 1 (nightly toolchain)".into(),
|
||||
reason:
|
||||
"sanitizers run on the accept path in verdict (ASan/UBSan, TSan when configured)"
|
||||
.into(),
|
||||
},
|
||||
));
|
||||
Ok(results)
|
||||
|
||||
Ok(GateReport {
|
||||
gates: results,
|
||||
fuzz_iters,
|
||||
fuzz_divergence,
|
||||
equivalence_mode: equivalence_mode.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn suite_manifest_derives_from_corpus_manifest() {
|
||||
assert_eq!(
|
||||
suite_manifest_path(Path::new("corpora/cjson/MANIFEST.sha256")),
|
||||
Path::new("corpora/cjson").join("TESTSUITE.sha256")
|
||||
);
|
||||
assert_eq!(
|
||||
suite_manifest_path(Path::new("MANIFEST.sha256")),
|
||||
Path::new("TESTSUITE.sha256")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_result_parses_last_exact_line() {
|
||||
assert_eq!(
|
||||
parse_fuzz_result("FUZZ-RESULT iters=10000 divergences=0\n"),
|
||||
Some((10000, 0))
|
||||
);
|
||||
// Last matching line wins; trailing non-matching output is ignored.
|
||||
let out = "FUZZ-RESULT iters=1 divergences=1\nprogress...\nFUZZ-RESULT iters=9999 divergences=2\ndone\n";
|
||||
assert_eq!(parse_fuzz_result(out), Some((9999, 2)));
|
||||
// CRLF-terminated capture still parses.
|
||||
assert_eq!(
|
||||
parse_fuzz_result("FUZZ-RESULT iters=42 divergences=0\r\n"),
|
||||
Some((42, 0))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_result_rejects_malformed_lines() {
|
||||
for bad in [
|
||||
"",
|
||||
"no result here",
|
||||
" FUZZ-RESULT iters=1 divergences=0", // leading space
|
||||
"FUZZ-RESULT iters=1 divergences=0", // double space
|
||||
"FUZZ-RESULT iters=abc divergences=0", // non-numeric
|
||||
"FUZZ-RESULT iters=1 divergences=-1", // sign not allowed
|
||||
"FUZZ-RESULT iters=1 divergences=0 xx", // trailing junk
|
||||
"FUZZ-RESULT divergences=0 iters=1", // wrong field order
|
||||
"FUZZ-RESULT iters=1", // missing field
|
||||
] {
|
||||
assert_eq!(parse_fuzz_result(bad), None, "should reject: {bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
fn inputs(baseline: Option<&'static str>) -> GateInputs<'static> {
|
||||
GateInputs {
|
||||
candidate_binary: "targets/cjson/cand/cjson-bench",
|
||||
baseline_binary: baseline,
|
||||
fuzz_iters: 10_000,
|
||||
}
|
||||
}
|
||||
|
||||
const TMPL: &str = "python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work \
|
||||
{iters} {baseline} {candidate}";
|
||||
|
||||
#[test]
|
||||
fn fuzz_command_substitutes_all_three_placeholders() {
|
||||
assert_eq!(
|
||||
fuzz_command(TMPL, 500, "base/bin", "cand/bin"),
|
||||
"python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work 500 base/bin cand/bin"
|
||||
);
|
||||
// Repeated placeholders all substitute; unknown braces are left alone.
|
||||
assert_eq!(
|
||||
fuzz_command("{candidate} {candidate} {iters} {other}", 2, "b", "c"),
|
||||
"c c 2 {other}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_skipped_when_no_command_declared() {
|
||||
assert_eq!(
|
||||
plan_fuzz(None, &inputs(Some("targets/cjson/baseline/cjson-bench"))),
|
||||
FuzzPlan::Skip("no fuzz command declared in target.toml [gates].fuzz")
|
||||
);
|
||||
// Absent command wins even without a baseline: nothing to run either way.
|
||||
assert_eq!(
|
||||
plan_fuzz(None, &inputs(None)),
|
||||
FuzzPlan::Skip(FUZZ_SKIP_NO_COMMAND)
|
||||
);
|
||||
}
|
||||
|
||||
/// The regression this branch exists for: `just gates` (and CI) build only
|
||||
/// the working tree, so there is no pristine baseline. Before the fix the
|
||||
/// fuzz script silently hardcoded targets/<t>/baseline/… — a path nothing
|
||||
/// on that flow builds — and the gate hard-failed. Now it skips, loudly.
|
||||
#[test]
|
||||
fn fuzz_skipped_when_declared_but_no_baseline() {
|
||||
assert_eq!(
|
||||
plan_fuzz(Some(TMPL), &inputs(None)),
|
||||
FuzzPlan::Skip(
|
||||
"differential fuzz needs a pristine baseline binary; run the accept path \
|
||||
(`just verdict`), which rebuilds one"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_runs_with_both_sides() {
|
||||
assert_eq!(
|
||||
plan_fuzz(
|
||||
Some(TMPL),
|
||||
&inputs(Some("targets/cjson/baseline/cjson-bench"))
|
||||
),
|
||||
FuzzPlan::Run(
|
||||
"python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work 10000 \
|
||||
targets/cjson/baseline/cjson-bench targets/cjson/cand/cjson-bench"
|
||||
.to_string()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_gate_grades_clean_run_as_pass() {
|
||||
let (outcome, iters, diverged) =
|
||||
fuzz_gate_outcome("FUZZ-RESULT iters=10000 divergences=0\n", true, Some(0));
|
||||
assert_eq!(outcome, GateOutcome::Passed);
|
||||
// The ledger records iterations *actually executed*, not the budget.
|
||||
assert_eq!(iters, 10_000);
|
||||
assert!(!diverged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzz_gate_fails_on_divergence_nonzero_exit_or_no_result_line() {
|
||||
// Divergence reported: fail, and the count is still recorded.
|
||||
let (outcome, iters, diverged) =
|
||||
fuzz_gate_outcome("FUZZ-RESULT iters=900 divergences=3\n", false, Some(1));
|
||||
assert!(
|
||||
matches!(outcome, GateOutcome::Failed { detail } if detail.contains("divergences=3"))
|
||||
);
|
||||
assert_eq!(iters, 900);
|
||||
assert!(diverged);
|
||||
|
||||
// Zero divergences but a nonzero exit (crash/timeout) is NOT a pass.
|
||||
let (outcome, ..) =
|
||||
fuzz_gate_outcome("FUZZ-RESULT iters=10 divergences=0\n", false, Some(2));
|
||||
assert!(
|
||||
matches!(outcome, GateOutcome::Failed { detail } if detail.contains("exit=Some(2)"))
|
||||
);
|
||||
|
||||
// A run that never reports the contract line fails closed (e.g. the
|
||||
// script died before printing — the old hardcoded-baseline breakage).
|
||||
let (outcome, iters, diverged) =
|
||||
fuzz_gate_outcome("Traceback (most recent call last):\n", false, Some(1));
|
||||
assert_eq!(
|
||||
outcome,
|
||||
GateOutcome::Failed {
|
||||
detail: "fuzz command did not report FUZZ-RESULT".into()
|
||||
}
|
||||
);
|
||||
assert_eq!(iters, 0);
|
||||
assert!(!diverged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fp_tolerance_requires_golden_reference() {
|
||||
let err = fp_reference(None).unwrap_err();
|
||||
assert_eq!(
|
||||
err,
|
||||
GateOutcome::Failed {
|
||||
detail: "fp-tolerance policy requires corpus.golden_reference in target.toml"
|
||||
.into()
|
||||
}
|
||||
);
|
||||
let path = PathBuf::from("corpora/matmul/GOLDEN.reference");
|
||||
assert_eq!(fp_reference(Some(&path)).unwrap(), &path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fp_golden_outcome_applies_policy() {
|
||||
let policy = EquivalencePolicy::FpTolerance {
|
||||
abs: 1e-6,
|
||||
rel: 1e-5,
|
||||
};
|
||||
// Last-ULP reordering within tolerance passes.
|
||||
assert_eq!(
|
||||
fp_golden_outcome(&policy, "1.0000000 2.0", "1.0000001 2.0", true, Some(0)),
|
||||
GateOutcome::Passed
|
||||
);
|
||||
// A genuine wrong value fails with the divergence in the detail.
|
||||
assert!(matches!(
|
||||
fp_golden_outcome(&policy, "1.0 2.0", "1.0 2.5", true, Some(0)),
|
||||
GateOutcome::Failed { detail } if detail.contains("token/offset 1")
|
||||
));
|
||||
// A failing replay command fails regardless of output.
|
||||
assert!(matches!(
|
||||
fp_golden_outcome(&policy, "1.0", "1.0", false, Some(3)),
|
||||
GateOutcome::Failed { detail } if detail.contains("exit=Some(3)")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,10 @@ pub struct TargetSpec {
|
||||
pub bench: Bench,
|
||||
pub gates: Gates,
|
||||
pub corpus: Corpus,
|
||||
/// Target name (the directory under `targets/`), captured by `load`;
|
||||
/// not part of target.toml itself.
|
||||
#[serde(skip)]
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -52,6 +56,14 @@ pub struct Build {
|
||||
/// Sanitizer binary path with `{out}` substituted.
|
||||
#[serde(default)]
|
||||
pub sanitizer_binary: Option<String>,
|
||||
/// TSan build command; `{out}` substituted. Run on the accept path
|
||||
/// when configured — a patch touching concurrency without a TSan
|
||||
/// lane cannot be machine-accepted.
|
||||
#[serde(default)]
|
||||
pub tsan: Option<String>,
|
||||
/// TSan binary path with `{out}` substituted.
|
||||
#[serde(default)]
|
||||
pub tsan_binary: Option<String>,
|
||||
}
|
||||
|
||||
/// Substitute the `{out}` isolation-dir placeholder in a build command or
|
||||
@@ -66,6 +78,11 @@ pub struct Gates {
|
||||
pub tests: String,
|
||||
/// Golden replay command; stdout is hashed. `{binary}` substituted.
|
||||
pub golden: String,
|
||||
/// Differential-fuzz command; `{candidate}` and `{iters}` substituted.
|
||||
/// Must print `FUZZ-RESULT iters=<n> divergences=<m>` as its final
|
||||
/// stdout line. Absent → the fuzz gate is skipped with a reason.
|
||||
#[serde(default)]
|
||||
pub fuzz: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -77,6 +94,11 @@ pub struct Corpus {
|
||||
/// File whose last whitespace-delimited-first-field line is the
|
||||
/// expected sha256 of the golden command's stdout.
|
||||
pub golden_sha256: PathBuf,
|
||||
/// Committed reference stdout of the golden command, repo-root-
|
||||
/// relative. Required by fp-tolerance targets: tolerance comparison
|
||||
/// needs the actual values, not a hash.
|
||||
#[serde(default)]
|
||||
pub golden_reference: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl TargetSpec {
|
||||
@@ -84,7 +106,15 @@ impl TargetSpec {
|
||||
let path = repo_root.join("targets").join(name).join("target.toml");
|
||||
let raw = std::fs::read_to_string(&path)
|
||||
.map_err(|e| anyhow::anyhow!("cannot read {}: {e}", path.display()))?;
|
||||
Ok(toml::from_str(&raw)?)
|
||||
let mut spec: Self = toml::from_str(&raw)?;
|
||||
spec.name = name.to_string();
|
||||
Ok(spec)
|
||||
}
|
||||
|
||||
/// This target's directory under the repo root (`targets/<name>`),
|
||||
/// where per-target policy files like equivalence.toml live.
|
||||
pub fn target_dir(&self, root: &Path) -> PathBuf {
|
||||
root.join("targets").join(&self.name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,3 +130,48 @@ pub fn expected_golden_hash(path: &Path) -> anyhow::Result<String> {
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| anyhow::anyhow!("no hash entry in {}", path.display()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn load_captures_name_and_defaults_new_fields() {
|
||||
let root = std::env::temp_dir().join("hotpath-target-load");
|
||||
let dir = root.join("targets").join("demo");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(
|
||||
dir.join("target.toml"),
|
||||
r#"
|
||||
[source]
|
||||
repo = "https://example.invalid/demo.git"
|
||||
commit = "deadbeef"
|
||||
license = "MIT"
|
||||
|
||||
[build]
|
||||
baseline = "make -C targets/demo OUT={out}"
|
||||
binary = "{out}/demo"
|
||||
|
||||
[bench]
|
||||
command = "{binary} bench"
|
||||
workload = "demo workload"
|
||||
|
||||
[gates]
|
||||
tests = "make -C targets/demo test"
|
||||
golden = "{binary} replay"
|
||||
|
||||
[corpus]
|
||||
manifest = "corpora/demo/MANIFEST.sha256"
|
||||
root = "corpora/demo"
|
||||
golden_sha256 = "corpora/demo/GOLDEN.sha256"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let spec = TargetSpec::load(&root, "demo").unwrap();
|
||||
assert_eq!(spec.target_dir(&root), root.join("targets").join("demo"));
|
||||
assert!(spec.gates.fuzz.is_none());
|
||||
assert!(spec.corpus.golden_reference.is_none());
|
||||
assert!(spec.build.tsan.is_none());
|
||||
assert!(spec.build.tsan_binary.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user