diff --git a/results/phase2/transcript-tokei-001.txt b/results/phase2/transcript-tokei-001.txt new file mode 100644 index 0000000..71c3be1 --- /dev/null +++ b/results/phase2/transcript-tokei-001.txt @@ -0,0 +1 @@ +[agent] Now let me see the outer driver `parse_lines` in `language_type.rs`. diff --git a/scripts/audit-attempt.py b/scripts/audit-attempt.py new file mode 100644 index 0000000..d321c8a --- /dev/null +++ b/scripts/audit-attempt.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Audit one ledger attempt (SPEC §10: 100% human audit in Phase 2). + +Prints the full evidence for a run and re-checks the verdict against the +accept rule mechanically: + + python3 scripts/audit-attempt.py [--threshold 0.02] + +Checks: + - verdict vs bench evidence: accepted requires ci_lower >= 1 + threshold + AND all gates green AND not flagged needs-human-review + - rejected-bench requires the CI to actually fail the bar + - patch paths stay inside the target workspace +The human part of the audit — does the patch express the hypothesis, is +the equivalence tier right — is printed for eyeballing, not automated. +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("run_id") + ap.add_argument("--threshold", type=float, default=0.02) + args = ap.parse_args() + + con = sqlite3.connect(ROOT / "results/ledger.sqlite") + con.row_factory = sqlite3.Row + row = con.execute("SELECT * FROM attempts WHERE run_id = ?", (args.run_id,)).fetchone() + if row is None: + print(f"no ledger row for {args.run_id}", file=sys.stderr) + return 2 + + gates = json.loads(row["gates"]) if row["gates"] else {} + bench = json.loads(row["bench"]) if row["bench"] else {} + verdict = row["verdict"] + + print(f"run_id : {row['run_id']}") + print(f"target : {row['target']} @ {row['target_commit']}") + print(f"class/hotspot: {row['playbook_class']} / {row['hotspot']}") + print(f"hypothesis : {row['hypothesis']}") + print(f"verdict : {verdict}") + print(f"gates : {json.dumps(gates, indent=2)}") + print(f"bench : {json.dumps(bench, indent=2)}") + + problems: list[str] = [] + ci = bench.get("speedup_ci") or [None, None] + ci_lo = ci[0] + bar = 1.0 + args.threshold + + if verdict == "accepted": + if ci_lo is None or ci_lo < bar: + problems.append(f"accepted but CI lower bound {ci_lo} < {bar}") + bad_gates = {k: v for k, v in gates.items() if v not in (True, "pass", "passed", "ok")} + if bad_gates: + problems.append(f"accepted with non-green gates: {bad_gates}") + elif verdict == "rejected-bench": + if ci_lo is not None and ci_lo >= bar: + problems.append(f"rejected-bench but CI lower bound {ci_lo} >= {bar}") + elif verdict == "needs-human-review": + pass # always a valid resting state; the ruling is ours + elif verdict == "rejected-gates": + if all(v in (True, "pass", "passed", "ok") for v in gates.values()) and gates: + problems.append("rejected-gates but every recorded gate is green") + + patch = row["patch"] or "" + for line in patch.splitlines(): + if line.startswith(("--- ", "+++ ")): + p = line[4:].strip() + if p.startswith("/") or "../" in p: + problems.append(f"patch path escapes workspace: {p}") + + print("\npatch:") + print(patch if patch else " (none recorded)") + + if problems: + print("\nAUDIT: PROBLEMS FOUND", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + print("\nAUDIT: mechanical checks clean (human eyeball still required)") + return 0 + + +if __name__ == "__main__": + sys.exit(main())