mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
gates: sanitizer check on the accept path — stop-the-line fix for phase2-comrak-010
The pipeline auto-accepted an LSan-flagged teardown patch (+10.7% median) because ASan/LSan were per-attempt manual; the 100% audit overturned it (black_box liveness ends at scope close, process::exit runs after — 384-byte arena chunk unreachable at exit). verdict now builds the patched tree with ASan+LSan and runs the pinned workload on every would-be accept, capping flagged wins at needs-human-review per SPEC §8. Shipped false accepts remain 0; the win was not banked; case study carries the corrective record. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
@@ -111,6 +111,59 @@ fn rebuild_pristine_baseline(
|
|||||||
Ok(baseline)
|
Ok(baseline)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// SPEC §8: an accept is final only if ASan+LSan are clean on the patched
|
||||||
|
/// tree over the pinned workload. Stop-the-line fix after phase2-comrak-010:
|
||||||
|
/// the pipeline auto-accepted an LSan-flagged teardown patch (sanitizers
|
||||||
|
/// were per-attempt manual) and the human audit had to overturn it — the
|
||||||
|
/// accept path now runs the check itself and caps flagged wins at
|
||||||
|
/// needs-human-review. Rejections skip it: they ship nothing.
|
||||||
|
fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result<bool> {
|
||||||
|
let ws = root.join(format!("targets/{target}/workspace"));
|
||||||
|
let asan_dir = root.join(format!("targets/{target}/asan"));
|
||||||
|
println!("sanitizers: ASan+LSan build (nightly) of the patched tree…");
|
||||||
|
let status = std::process::Command::new("cargo")
|
||||||
|
.args([
|
||||||
|
"+nightly",
|
||||||
|
"build",
|
||||||
|
"-q",
|
||||||
|
"--target",
|
||||||
|
"x86_64-unknown-linux-gnu",
|
||||||
|
])
|
||||||
|
.env("RUSTFLAGS", "-Zsanitizer=address")
|
||||||
|
.env("CARGO_TARGET_DIR", &asan_dir)
|
||||||
|
.current_dir(&ws)
|
||||||
|
.status()?;
|
||||||
|
anyhow::ensure!(status.success(), "sanitizer build failed");
|
||||||
|
let bin_name = std::path::Path::new(&spec.build.binary)
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
||||||
|
let asan_bin = asan_dir.join(format!("x86_64-unknown-linux-gnu/debug/{bin_name}"));
|
||||||
|
let cmd = spec
|
||||||
|
.bench
|
||||||
|
.command
|
||||||
|
.replace("{binary}", asan_bin.to_str().unwrap());
|
||||||
|
let out = std::process::Command::new("sh")
|
||||||
|
.arg("-c")
|
||||||
|
.arg(&cmd)
|
||||||
|
.env("ASAN_OPTIONS", "detect_leaks=1")
|
||||||
|
.current_dir(root)
|
||||||
|
.output()?;
|
||||||
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
|
let clean = out.status.success() && !stderr.contains("ERROR:");
|
||||||
|
if clean {
|
||||||
|
println!("sanitizers: clean");
|
||||||
|
} else {
|
||||||
|
let lines: Vec<&str> = stderr.lines().filter(|l| !l.is_empty()).collect();
|
||||||
|
let start = lines.len().saturating_sub(8);
|
||||||
|
println!("sanitizers: FLAGGED (rc={:?})", out.status.code());
|
||||||
|
for l in &lines[start..] {
|
||||||
|
println!(" {l}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(clean)
|
||||||
|
}
|
||||||
|
|
||||||
fn now_utc() -> String {
|
fn now_utc() -> String {
|
||||||
let out = std::process::Command::new("date")
|
let out = std::process::Command::new("date")
|
||||||
.args(["-u", "+%Y-%m-%dT%H:%M:%SZ"])
|
.args(["-u", "+%Y-%m-%dT%H:%M:%SZ"])
|
||||||
@@ -146,7 +199,7 @@ fn main() -> Result<()> {
|
|||||||
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
|
GateOutcome::Skipped { reason } => println!("{layer:?}: skipped — {reason}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let gate_results = GateResults {
|
let mut gate_results = GateResults {
|
||||||
upstream_tests: matches!(gates[0].1, GateOutcome::Passed),
|
upstream_tests: matches!(gates[0].1, GateOutcome::Passed),
|
||||||
golden_replay: matches!(gates[1].1, GateOutcome::Passed),
|
golden_replay: matches!(gates[1].1, GateOutcome::Passed),
|
||||||
fuzz_iters: 0,
|
fuzz_iters: 0,
|
||||||
@@ -165,8 +218,7 @@ fn main() -> Result<()> {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let (verdict, bench) =
|
let (verdict, bench) = if !all_passed(&gates) {
|
||||||
if !all_passed(&gates) {
|
|
||||||
println!("verdict: rejected-gate (bench skipped)");
|
println!("verdict: rejected-gate (bench skipped)");
|
||||||
(Verdict::RejectedGate, None)
|
(Verdict::RejectedGate, None)
|
||||||
} else {
|
} else {
|
||||||
@@ -202,12 +254,24 @@ fn main() -> Result<()> {
|
|||||||
);
|
);
|
||||||
println!(
|
println!(
|
||||||
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
|
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}] | workload: {}",
|
||||||
ratio.median, cfg.confidence * 100.0, ratio.lo, ratio.hi, spec.bench.workload
|
ratio.median,
|
||||||
|
cfg.confidence * 100.0,
|
||||||
|
ratio.lo,
|
||||||
|
ratio.hi,
|
||||||
|
spec.bench.workload
|
||||||
);
|
);
|
||||||
let mut v = decide(ratio, cfg.threshold);
|
let mut v = decide(ratio, cfg.threshold);
|
||||||
if cli.needs_human_review && v == Verdict::Accepted {
|
if cli.needs_human_review && v == Verdict::Accepted {
|
||||||
v = Verdict::NeedsHumanReview;
|
v = Verdict::NeedsHumanReview;
|
||||||
}
|
}
|
||||||
|
if v == Verdict::Accepted {
|
||||||
|
let clean = sanitizer_check(&root, &cli.target, &spec)?;
|
||||||
|
gate_results.sanitizers_clean = clean;
|
||||||
|
if !clean {
|
||||||
|
println!("verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)");
|
||||||
|
v = Verdict::NeedsHumanReview;
|
||||||
|
}
|
||||||
|
}
|
||||||
let fp = EnvFingerprint::collect(pin, "system-default");
|
let fp = EnvFingerprint::collect(pin, "system-default");
|
||||||
(
|
(
|
||||||
v,
|
v,
|
||||||
@@ -222,7 +286,7 @@ fn main() -> Result<()> {
|
|||||||
"fingerprint": fp,
|
"fingerprint": fp,
|
||||||
"workload": spec.bench.workload,
|
"workload": spec.bench.workload,
|
||||||
"target_commit": spec.source.commit,
|
"target_commit": spec.source.commit,
|
||||||
"gates_detail": "fuzz/sanitizers per-attempt manual in Phase 1",
|
"gates_detail": "diff-fuzz per-attempt manual; ASan+LSan enforced on the accept path",
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -89,6 +89,31 @@ through `propose_patch` (path-allowlist + `git apply`) and the full
|
|||||||
gate+bench pipeline. Both were correctly rejected — real mechanisms,
|
gate+bench pipeline. Both were correctly rejected — real mechanisms,
|
||||||
too small to clear the 2% CI-lower-bound bar. **No false accepts.**
|
too small to clear the 2% CI-lower-bound bar. **No false accepts.**
|
||||||
|
|
||||||
|
## The stop-the-line event: phase2-comrak-010
|
||||||
|
|
||||||
|
The pipeline auto-accepted a comrak class-3 patch (+10.7% median,
|
||||||
|
CI [+7.9%, +12.2%]) that skips AST-arena teardown via `ManuallyDrop`
|
||||||
|
before `process::exit` — and the 100% human audit **overturned it**.
|
||||||
|
LeakSanitizer flags a 384-byte arena chunk as unreachable at exit: the
|
||||||
|
patch's `black_box(&arena)` keeps the arena live only until its scope
|
||||||
|
closes, but `process::exit` runs after. The agent's own hypothesis had
|
||||||
|
named this exact risk ("if the sanitizer still flags the leak,
|
||||||
|
rejected-gates is the honest outcome") — the pipeline just had no
|
||||||
|
sanitizer gate to trip: ASan/LSan were per-attempt manual, so the row
|
||||||
|
says `accepted` with `sanitizers_clean: false`.
|
||||||
|
|
||||||
|
Called per SPEC §10: a **pipeline false accept**, caught by audit before
|
||||||
|
anything shipped (shipped false accepts remain **0**; the win was not
|
||||||
|
banked). Fix shipped the same hour: `verdict` now runs an ASan+LSan
|
||||||
|
build of the patched tree over the pinned workload on every would-be
|
||||||
|
accept and caps flagged wins at `needs-human-review` — the tier rule
|
||||||
|
SPEC §8 always specified, now enforced by the machine instead of the
|
||||||
|
auditor. The immutable ledger row stands as written; this section is
|
||||||
|
the corrective record.
|
||||||
|
|
||||||
|
10k-input differential fuzz on the same patch: 0 divergences — the
|
||||||
|
output equivalence was real; only the teardown-leak tier was wrong.
|
||||||
|
|
||||||
## Two findings the loop surfaced about itself
|
## Two findings the loop surfaced about itself
|
||||||
|
|
||||||
1. **`run_verdict` outlived the MCP transport (attempt 001).** The
|
1. **`run_verdict` outlived the MCP transport (attempt 001).** The
|
||||||
|
|||||||
Reference in New Issue
Block a user