mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
gates: sanitizer check on the accept path — stop-the-line fix for phase2-comrak-010
The pipeline auto-accepted an LSan-flagged teardown patch (+10.7% median) because ASan/LSan were per-attempt manual; the 100% audit overturned it (black_box liveness ends at scope close, process::exit runs after — 384-byte arena chunk unreachable at exit). verdict now builds the patched tree with ASan+LSan and runs the pinned workload on every would-be accept, capping flagged wins at needs-human-review per SPEC §8. Shipped false accepts remain 0; the win was not banked; case study carries the corrective record. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
@@ -111,6 +111,59 @@ fn rebuild_pristine_baseline(
|
||||
Ok(baseline)
|
||||
}
|
||||
|
||||
/// SPEC §8: an accept is final only if ASan+LSan are clean on the patched
|
||||
/// tree over the pinned workload. Stop-the-line fix after phase2-comrak-010:
|
||||
/// the pipeline auto-accepted an LSan-flagged teardown patch (sanitizers
|
||||
/// were per-attempt manual) and the human audit had to overturn it — the
|
||||
/// accept path now runs the check itself and caps flagged wins at
|
||||
/// needs-human-review. Rejections skip it: they ship nothing.
|
||||
fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result<bool> {
|
||||
let ws = root.join(format!("targets/{target}/workspace"));
|
||||
let asan_dir = root.join(format!("targets/{target}/asan"));
|
||||
println!("sanitizers: ASan+LSan build (nightly) of the patched tree…");
|
||||
let status = std::process::Command::new("cargo")
|
||||
.args([
|
||||
"+nightly",
|
||||
"build",
|
||||
"-q",
|
||||
"--target",
|
||||
"x86_64-unknown-linux-gnu",
|
||||
])
|
||||
.env("RUSTFLAGS", "-Zsanitizer=address")
|
||||
.env("CARGO_TARGET_DIR", &asan_dir)
|
||||
.current_dir(&ws)
|
||||
.status()?;
|
||||
anyhow::ensure!(status.success(), "sanitizer build failed");
|
||||
let bin_name = std::path::Path::new(&spec.build.binary)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
||||
let asan_bin = asan_dir.join(format!("x86_64-unknown-linux-gnu/debug/{bin_name}"));
|
||||
let cmd = spec
|
||||
.bench
|
||||
.command
|
||||
.replace("{binary}", asan_bin.to_str().unwrap());
|
||||
let out = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(&cmd)
|
||||
.env("ASAN_OPTIONS", "detect_leaks=1")
|
||||
.current_dir(root)
|
||||
.output()?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let clean = out.status.success() && !stderr.contains("ERROR:");
|
||||
if clean {
|
||||
println!("sanitizers: clean");
|
||||
} else {
|
||||
let lines: Vec<&str> = stderr.lines().filter(|l| !l.is_empty()).collect();
|
||||
let start = lines.len().saturating_sub(8);
|
||||
println!("sanitizers: FLAGGED (rc={:?})", out.status.code());
|
||||
for l in &lines[start..] {
|
||||
println!(" {l}");
|
||||
}
|
||||
}
|
||||
Ok(clean)
|
||||
}
|
||||
|
||||
fn now_utc() -> String {
|
||||
let out = std::process::Command::new("date")
|
||||
.args(["-u", "+%Y-%m-%dT%H:%M:%SZ"])
|
||||
@@ -146,7 +199,7 @@ fn main() -> Result<()> {
|
||||
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
|
||||
}
|
||||
}
|
||||
let gate_results = GateResults {
|
||||
let mut gate_results = GateResults {
|
||||
upstream_tests: matches!(gates[0].1, GateOutcome::Passed),
|
||||
golden_replay: matches!(gates[1].1, GateOutcome::Passed),
|
||||
fuzz_iters: 0,
|
||||
@@ -165,8 +218,7 @@ fn main() -> Result<()> {
|
||||
}
|
||||
};
|
||||
|
||||
let (verdict, bench) =
|
||||
if !all_passed(&gates) {
|
||||
let (verdict, bench) = if !all_passed(&gates) {
|
||||
println!("verdict: rejected-gate (bench skipped)");
|
||||
(Verdict::RejectedGate, None)
|
||||
} else {
|
||||
@@ -202,12 +254,24 @@ fn main() -> Result<()> {
|
||||
);
|
||||
println!(
|
||||
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
|
||||
ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload
|
||||
ratio.median,
|
||||
cfg.confidence * 100.0,
|
||||
ratio.lo,
|
||||
ratio.hi,
|
||||
spec.bench.workload
|
||||
);
|
||||
let mut v = decide(ratio, cfg.threshold);
|
||||
if cli.needs_human_review && v == Verdict::Accepted {
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
if v == Verdict::Accepted {
|
||||
let clean = sanitizer_check(&root, &cli.target, &spec)?;
|
||||
gate_results.sanitizers_clean = clean;
|
||||
if !clean {
|
||||
println!("verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)");
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
}
|
||||
let fp = EnvFingerprint::collect(pin, "system-default");
|
||||
(
|
||||
v,
|
||||
@@ -222,7 +286,7 @@ fn main() -> Result<()> {
|
||||
"fingerprint": fp,
|
||||
"workload": spec.bench.workload,
|
||||
"target_commit": spec.source.commit,
|
||||
"gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
|
||||
"gates_detail": "diff-fuzz per-attempt manual; ASan+LSan enforced on the accept path",
|
||||
}),
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -89,6 +89,31 @@ through `propose_patch` (path-allowlist + `git apply`) and the full
|
||||
gate+bench pipeline. Both were correctly rejected — real mechanisms,
|
||||
too small to clear the 2% CI-lower-bound bar. **No false accepts.**
|
||||
|
||||
## The stop-the-line event: phase2-comrak-010
|
||||
|
||||
The pipeline auto-accepted a comrak class-3 patch (+10.7% median,
|
||||
CI [+7.9%, +12.2%]) that skips AST-arena teardown via `ManuallyDrop`
|
||||
before `process::exit` — and the 100% human audit **overturned it**.
|
||||
LeakSanitizer flags a 384-byte arena chunk as unreachable at exit: the
|
||||
patch's `black_box(&arena)` keeps the arena live only until its scope
|
||||
closes, but `process::exit` runs after. The agent's own hypothesis had
|
||||
named this exact risk ("if the sanitizer still flags the leak,
|
||||
rejected-gates is the honest outcome") — the pipeline just had no
|
||||
sanitizer gate to trip: ASan/LSan were per-attempt manual, so the row
|
||||
says `accepted` with `sanitizers_clean: false`.
|
||||
|
||||
Called per SPEC §10: a **pipeline false accept**, caught by audit before
|
||||
anything shipped (shipped false accepts remain **0**; the win was not
|
||||
banked). Fix shipped the same hour: `verdict` now runs an ASan+LSan
|
||||
build of the patched tree over the pinned workload on every would-be
|
||||
accept and caps flagged wins at `needs-human-review` — the tier rule
|
||||
SPEC §8 always specified, now enforced by the machine instead of the
|
||||
auditor. The immutable ledger row stands as written; this section is
|
||||
the corrective record.
|
||||
|
||||
10k-input differential fuzz on the same patch: 0 divergences — the
|
||||
output equivalence was real; only the teardown-leak tier was wrong.
|
||||
|
||||
## Two findings the loop surfaced about itself
|
||||
|
||||
1. **`run_verdict` outlived the MCP transport (attempt 001).** The
|
||||
|
||||
Reference in New Issue
Block a user