From eaadafc5c9be37031f615b42255fc7bc10c070a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 12:26:30 +0000 Subject: [PATCH] =?UTF-8?q?gates:=20sanitizer=20check=20on=20the=20accept?= =?UTF-8?q?=20path=20=E2=80=94=20stop-the-line=20fix=20for=20phase2-comrak?= =?UTF-8?q?-010?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline auto-accepted an LSan-flagged teardown patch (+10.7% median) because ASan/LSan were per-attempt manual; the 100% audit overturned it (black_box liveness ends at scope close, process::exit runs after — 384-byte arena chunk unreachable at exit). verdict now builds the patched tree with ASan+LSan and runs the pinned workload on every would-be accept, capping flagged wins at needs-human-review per SPEC §8. Shipped false accepts remain 0; the win was not banked; case study carries the corrective record. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2 --- crates/verdict/src/main.rs | 182 ++++++++++++++++-------- results/phase2/case-study-agent-loop.md | 25 ++++ 2 files changed, 148 insertions(+), 59 deletions(-) diff --git a/crates/verdict/src/main.rs b/crates/verdict/src/main.rs index 5ed1dfe..3392ed9 100644 --- a/crates/verdict/src/main.rs +++ b/crates/verdict/src/main.rs @@ -111,6 +111,59 @@ fn rebuild_pristine_baseline( Ok(baseline) } +/// SPEC §8: an accept is final only if ASan+LSan are clean on the patched +/// tree over the pinned workload. Stop-the-line fix after phase2-comrak-010: +/// the pipeline auto-accepted an LSan-flagged teardown patch (sanitizers +/// were per-attempt manual) and the human audit had to overturn it — the +/// accept path now runs the check itself and caps flagged wins at +/// needs-human-review. Rejections skip it: they ship nothing. +fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result { + let ws = root.join(format!("targets/{target}/workspace")); + let asan_dir = root.join(format!("targets/{target}/asan")); + println!("sanitizers: ASan+LSan build (nightly) of the patched tree…"); + let status = std::process::Command::new("cargo") + .args([ + "+nightly", + "build", + "-q", + "--target", + "x86_64-unknown-linux-gnu", + ]) + .env("RUSTFLAGS", "-Zsanitizer=address") + .env("CARGO_TARGET_DIR", &asan_dir) + .current_dir(&ws) + .status()?; + anyhow::ensure!(status.success(), "sanitizer build failed"); + let bin_name = std::path::Path::new(&spec.build.binary) + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?; + let asan_bin = asan_dir.join(format!("x86_64-unknown-linux-gnu/debug/{bin_name}")); + let cmd = spec + .bench + .command + .replace("{binary}", asan_bin.to_str().unwrap()); + let out = std::process::Command::new("sh") + .arg("-c") + .arg(&cmd) + .env("ASAN_OPTIONS", "detect_leaks=1") + .current_dir(root) + .output()?; + let stderr = String::from_utf8_lossy(&out.stderr); + let clean = out.status.success() && !stderr.contains("ERROR:"); + if clean { + println!("sanitizers: clean"); + } else { + let lines: Vec<&str> = stderr.lines().filter(|l| !l.is_empty()).collect(); + let start = lines.len().saturating_sub(8); + println!("sanitizers: FLAGGED (rc={:?})", out.status.code()); + for l in &lines[start..] { + println!(" {l}"); + } + } + Ok(clean) +} + fn now_utc() -> String { let out = std::process::Command::new("date") .args(["-u", "+%Y-%m-%dT%H:%M:%SZ"]) @@ -146,7 +199,7 @@ fn main() -> Result<()> { GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"), } } - let gate_results = GateResults { + let mut gate_results = GateResults { upstream_tests: matches!(gates[0].1, GateOutcome::Passed), golden_replay: matches!(gates[1].1, GateOutcome::Passed), fuzz_iters: 0, @@ -165,68 +218,79 @@ fn main() -> Result<()> { } }; - let (verdict, bench) = - if !all_passed(&gates) { - println!("verdict: rejected-gate (bench skipped)"); - (Verdict::RejectedGate, None) - } else { - println!( - "bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'", - cfg.runs_per_side, cfg.warmup_runs, pin - ); - let samples = exec::run_interleaved( - &wrap(&baseline_bin), - &wrap(&cli.candidate_bin), - cfg.runs_per_side, - cfg.warmup_runs, - 0.0, - )?; - let ratio = stats::bootstrap_ratio_ci( - &samples.baseline_s, - &samples.candidate_s, - cfg.bootstrap_iters, - cfg.confidence, - cfg.bootstrap_seed, - ); - let (bm, blo, bhi) = stats::bootstrap_median_ci( - &samples.baseline_s, - cfg.bootstrap_iters, - cfg.confidence, - cfg.bootstrap_seed, - ); - let (cm, clo, chi) = stats::bootstrap_median_ci( - &samples.candidate_s, - cfg.bootstrap_iters, - cfg.confidence, - cfg.bootstrap_seed, - ); - println!( - "speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}", - ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload + let (verdict, bench) = if !all_passed(&gates) { + println!("verdict: rejected-gate (bench skipped)"); + (Verdict::RejectedGate, None) + } else { + println!( + "bench: {} measured + {} warm-up runs/side, interleaved, pin='{}'", + cfg.runs_per_side, cfg.warmup_runs, pin ); - let mut v = decide(ratio, cfg.threshold); - if cli.needs_human_review && v == Verdict::Accepted { + let samples = exec::run_interleaved( + &wrap(&baseline_bin), + &wrap(&cli.candidate_bin), + cfg.runs_per_side, + cfg.warmup_runs, + 0.0, + )?; + let ratio = stats::bootstrap_ratio_ci( + &samples.baseline_s, + &samples.candidate_s, + cfg.bootstrap_iters, + cfg.confidence, + cfg.bootstrap_seed, + ); + let (bm, blo, bhi) = stats::bootstrap_median_ci( + &samples.baseline_s, + cfg.bootstrap_iters, + cfg.confidence, + cfg.bootstrap_seed, + ); + let (cm, clo, chi) = stats::bootstrap_median_ci( + &samples.candidate_s, + cfg.bootstrap_iters, + cfg.confidence, + cfg.bootstrap_seed, + ); + println!( + "speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}", + ratio.median, + cfg.confidence * 100.0, + ratio.lo, + ratio.hi, + spec.bench.workload + ); + let mut v = decide(ratio, cfg.threshold); + if cli.needs_human_review && v == Verdict::Accepted { + v = Verdict::NeedsHumanReview; + } + if v == Verdict::Accepted { + let clean = sanitizer_check(&root, &cli.target, &spec)?; + gate_results.sanitizers_clean = clean; + if !clean { + println!("verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)"); v = Verdict::NeedsHumanReview; } - let fp = EnvFingerprint::collect(pin, "system-default"); - ( - v, - Some(BenchEvidence { - baseline_median: bm, - baseline_ci: (blo, bhi), - candidate_median: cm, - candidate_ci: (clo, chi), - speedup_median: ratio.median, - speedup_ci: (ratio.lo, ratio.hi), - env_fingerprint: serde_json::json!({ - "fingerprint": fp, - "workload": spec.bench.workload, - "target_commit": spec.source.commit, - "gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1", - }), + } + let fp = EnvFingerprint::collect(pin, "system-default"); + ( + v, + Some(BenchEvidence { + baseline_median: bm, + baseline_ci: (blo, bhi), + candidate_median: cm, + candidate_ci: (clo, chi), + speedup_median: ratio.median, + speedup_ci: (ratio.lo, ratio.hi), + env_fingerprint: serde_json::json!({ + "fingerprint": fp, + "workload": spec.bench.workload, + "target_commit": spec.source.commit, + "gates_detail": "diff-fuzz per-attempt manual; ASan+LSan enforced on the accept path", }), - ) - }; + }), + ) + }; let patch = match &cli.patch_file { Some(p) => std::fs::read_to_string(p)?, diff --git a/results/phase2/case-study-agent-loop.md b/results/phase2/case-study-agent-loop.md index 0f08446..fdb4cbb 100644 --- a/results/phase2/case-study-agent-loop.md +++ b/results/phase2/case-study-agent-loop.md @@ -89,6 +89,31 @@ through `propose_patch` (path-allowlist + `git apply`) and the full gate+bench pipeline. Both were correctly rejected — real mechanisms, too small to clear the 2% CI-lower-bound bar. **No false accepts.** +## The stop-the-line event: phase2-comrak-010 + +The pipeline auto-accepted a comrak class-3 patch (+10.7% median, +CI [+7.9%, +12.2%]) that skips AST-arena teardown via `ManuallyDrop` +before `process::exit` — and the 100% human audit **overturned it**. +LeakSanitizer flags a 384-byte arena chunk as unreachable at exit: the +patch's `black_box(&arena)` keeps the arena live only until its scope +closes, but `process::exit` runs after. The agent's own hypothesis had +named this exact risk ("if the sanitizer still flags the leak, +rejected-gates is the honest outcome") — the pipeline just had no +sanitizer gate to trip: ASan/LSan were per-attempt manual, so the row +says `accepted` with `sanitizers_clean: false`. + +Called per SPEC §10: a **pipeline false accept**, caught by audit before +anything shipped (shipped false accepts remain **0**; the win was not +banked). Fix shipped the same hour: `verdict` now runs an ASan+LSan +build of the patched tree over the pinned workload on every would-be +accept and caps flagged wins at `needs-human-review` — the tier rule +SPEC §8 always specified, now enforced by the machine instead of the +auditor. The immutable ledger row stands as written; this section is +the corrective record. + +10k-input differential fuzz on the same patch: 0 divergences — the +output equivalence was real; only the teardown-leak tier was wrong. + ## Two findings the loop surfaced about itself 1. **`run_verdict` outlived the MCP transport (attempt 001).** The