Merge pull request #14 from bdbrown4/claude/test-wsk42i

Phase 5 research forks + adversarial review, mimalloc ruling, and docs site
2026-11-07 10:58:28 +00:00 · 2026-07-11 20:41:05 -04:00
parent d1ef2a0dd6 ff16e67127
commit 94b3ab9154
47 changed files with 1934 additions and 56 deletions
@@ -0,0 +1,53 @@
 name: docs
 # Builds the mdBook site in docs/ and deploys it to GitHub Pages.
 # One-time setup: repo Settings → Pages → Source = "GitHub Actions".
 on:
  push:
    branches: [main]
    paths:
      - "docs/**"
      - ".github/workflows/docs.yml"
  workflow_dispatch:
 permissions:
  contents: read
  pages: write
  id-token: write
 # Allow one concurrent deployment; don't cancel an in-progress prod deploy.
 concurrency:
  group: pages
  cancel-in-progress: false
 jobs:
  build:
    runs-on: ubuntu-latest
    env:
      MDBOOK_VERSION: "0.4.40"
    steps:
      - uses: actions/checkout@v4
      - name: Install mdBook
        run: |
          mkdir -p "$HOME/.local/bin"
          url="https://github.com/rust-lang/mdBook/releases/download/v${MDBOOK_VERSION}/mdbook-v${MDBOOK_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
          curl -sSL "$url" | tar -xz -C "$HOME/.local/bin"
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
      - name: Build
        run: mdbook build docs
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v3
        with:
          path: docs/book
  deploy:
    needs: build
    runs-on: ubuntu-latest
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4
@@ -249,3 +249,4 @@ targets/cjson/cand-*/
 targets/cjson/asan*/
 targets/cjson/svc-baseline/
 targets/cjson/svc-candidate/
 docs/book/
@@ -30,7 +30,9 @@ Full engineering spec: `SPEC.md`. Read it before starting any phase.
 - `playbook/` — optimization classes, ordered, with preconditions and known risks
 - `results/` — generated reports and flamegraphs
-**Phase 4 COMPLETE — all SPEC §5 exit criteria met. Phases 0–4 are all done.** Service-mode latency benchmarking shipped: a coordinated-omission-correct, open-loop, fixed-rate load generator (`crates/bench-runner/src/service.rs`) measuring latency from each request's intended send time (queueing counted, not hidden), interleaved A/B, exact percentiles, bootstrap p50/p99 CIs. A real service target — `targets/cjson/service.c`, a minimal HTTP server wrapping the patched `cJSON.c` (trust-layer, outside the workspace allowlist). Service calibration PASS (0/10 A/A false-positive, 10/10 injected-5%-latency-regression detection; `results/calibration/cjson-service-aa.json`). **The verified batch win phase3-cjson-002 measured under a 150 rps replay (20 rounds, 40k requests, 0 drops): p50 latency +6.2%, 95% CI [+5.8%, +7.2%] — accepted (`phase4-cjson-service` ledger row); p99 NOT claimed (CI [0.07, 4.97], single-worker loopback tail jitter) — harness correctly rejected it.** Mechanical ROI report generator (`crates/report` bin, `just report`, SPEC §9): reads a ledger row + service-latency JSON → throughput→cores→dollars (27.5 cores / $9,621-per-year CI lower bound on a 500-core fleet) *and* latency percentiles, every figure with CI + workload + methodology inline; flags any non-accepted row. Case study: `results/phase4/`. **The roadmap is complete — the profile→hypothesize→patch→gated-verdict→ROI loop runs end to end across Rust and C, batch and service, with zero shipped false accepts across all phases. Open follow-ups: the standing human ruling on `phase0-comrak-002` (mimalloc, needs-human-review); SPEC §13 research forks (GPU lane, learned policies) remain out of scope.**
+**Phase 4 COMPLETE — all SPEC §5 exit criteria met. Phases 0–4 are all done.** Service-mode latency benchmarking shipped: a coordinated-omission-correct, open-loop, fixed-rate load generator (`crates/bench-runner/src/service.rs`) measuring latency from each request's intended send time (queueing counted, not hidden), interleaved A/B, exact percentiles, bootstrap p50/p99 CIs. A real service target — `targets/cjson/service.c`, a minimal HTTP server wrapping the patched `cJSON.c` (trust-layer, outside the workspace allowlist). Service calibration PASS (0/10 A/A false-positive, 10/10 injected-5%-latency-regression detection; `results/calibration/cjson-service-aa.json`). **The verified batch win phase3-cjson-002 measured under a 150 rps replay (20 rounds, 40k requests, 0 drops): p50 latency +6.2%, 95% CI [+5.8%, +7.2%] — accepted (`phase4-cjson-service` ledger row); p99 NOT claimed (CI [0.07, 4.97], single-worker loopback tail jitter) — harness correctly rejected it.** Mechanical ROI report generator (`crates/report` bin, `just report`, SPEC §9): reads a ledger row + service-latency JSON → throughput→cores→dollars (27.5 cores / $9,621-per-year CI lower bound on a 500-core fleet) *and* latency percentiles, every figure with CI + workload + methodology inline; flags any non-accepted row. Case study: `results/phase4/`. **The roadmap is complete — the profile→hypothesize→patch→gated-verdict→ROI loop runs end to end across Rust and C, batch and service, with zero shipped false accepts across all phases.**
 **Phase 5 COMPLETE — the two SPEC §13 research forks, both built and both routed to human review.** (a) *Learned class-selection policy* (`crates/policy`, `cargo run -p policy`): reads the append-only ledger and ranks optimization classes by the Wilson lower bound of their shippable-win rate — a learned prior over the fixed cheapest-first ordering, advisory only (gates still decide). A "win" counts only machine-sanitizer-verified accepts; the overturned comrak-010 and tier-gated mimalloc are excluded/held. On the current 34-row ledger it recommends algorithmic first (Wilson lb 0.066). (b) *Kernel lane* (`targets/matmul/kernel.c`, `crates/diff-test` FP-tolerance policy + `fp-compare` bin, `scripts/kernel-lane-demo.sh`): a matmul optimization (transpose-B cache locality + eight-accumulator ILP) that **reorders FP accumulation**, so byte-identical golden replay is the wrong gate — 244,901/262,144 values differ. The FP-tolerance gate (`abs 1e-4 + rel 1e-3`) accepts it, still REJECTS a +0.5 perturbation, and the bench measures 3.23× [3.16, 3.26] with the same interleaved A/B + bootstrap CI. Ledger row `phase5-matmul-opt` = needs-human-review (using the tolerance tier is a §8 signal, same posture as mimalloc). No GPU in this environment, so the GPU fork is the identical shape shown on CPU — only the timer and hardware change. Case study: `results/phase5/`; docs chapter `docs/src/research-forks.md`. Open follow-up: the standing accept-scoped ruling on `phase0-comrak-002` (mimalloc) is resolved (`results/rulings/`).**
 ## Commands
@@ -351,6 +351,15 @@ version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 [[package]]
 name = "policy"
 version = "0.1.0"
 dependencies = [
 "anyhow",
 "clap",
 "ledger",
 ]
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"
@@ -9,6 +9,7 @@ members = [
    "crates/report",
    "crates/verdict",
    "crates/harnessd",
    "crates/policy",
 ]
 [workspace.package]
@@ -1,72 +1,88 @@
-# peltier.io
+# hotpath
-Spot cooling for hot paths. A profile-guided optimization agent that only ships wins it can prove.
+> Spot cooling for hot paths. A profile-guided optimization agent that
 > only ships wins it can prove.
-hotpath finds the 10–90% left on the floor between "the compiler" and
+hotpath profiles real binaries and services, isolates hot paths, proposes
-"hire a perf consultant," with two properties no consultant offers:
+optimizations, and accepts a change **only** when two independent bars are
-**verified equivalence** (a change provably preserves behavior or is
+cleared: **behavioral equivalence** (upstream tests + byte-identical golden
-discarded) and **self-verifying ROI** (the value claim is a stopwatch on
+replay + differential fuzzing + sanitizers) and **statistical significance**
-trusted infrastructure, not a survey).
+(the speedup's bootstrap 95% CI lower bound clears a threshold on
 calibrated hardware). A change without both is discarded and logged.
-Full design: **[SPEC.md](SPEC.md)**.
+Consultants sell surveys; compilers sell flags. hotpath sells **verified
 deltas with the methodology attached** — a stopwatch on pinned hardware, a
 proof of equivalence, and an ROI figure whose confidence interval survives
 hostile review.
 📖 **Full documentation: the [hotpath book](https://bdbrown4.github.io/peltier.io/)**
 · Design spec: [SPEC.md](SPEC.md) · Charter: [CLAUDE.md](CLAUDE.md)
 ## Verified results
 Every number carries its 95% CI and its workload, and every one survived the
 gates above. Across **five phases and 33 ledger rows, zero shipped false
 accepts** — including two pipeline over-accepts the audit caught before
 anything shipped, each becoming a permanent new gate.
 | Target | Win | Verified |
 |---|---|---|
 | tokei (Rust) — LTO (class 1) | +10.4% | CI [+8.5%, +12.0%] |
 | tokei — three class-5 algorithmic wins | +10.7% / +3.7% / +9.9% | each CI-significant |
 | cJSON (C) — number-handling rewrite (class 5) | +8.85% | CI [+7.5%, +10.0%] |
 | cJSON HTTP service — p50 latency under replay | +6.2% | CI [+5.8%, +7.2%] |
 | comrak (Rust) — mimalloc (class 2) | +4.6% | CI [+3.3%, +5.8%], [human-ruled](results/rulings/phase0-comrak-002.md) |
 ## Layout
 ```
-crates/            Trust layer (agent has NO write access — SPEC §10)
+crates/            Trust layer — agent has NO write access (SPEC §10)
-  bench-runner/    Interleaved A/B timing, bootstrap CIs, A/A calibration
+  bench-runner/    Interleaved A/B, bootstrap CIs, A/A calibration, service mode
-  diff-test/       Equivalence gates, corpus hash-pinning, per-target policy
+  diff-test/       Equivalence gates, corpus hash-pinning, per-target spec
-  ledger/          Append-only SQLite attempt ledger (enforced by triggers)
+  ledger/          Append-only SQLite attempt ledger (mutation-refusing triggers)
-  report/          ROI math: speedup CI → cores → dollars, caveats included
+  report/          ROI: speedup CI → cores → dollars, methodology inline
  verdict/         The pipeline in one command: gates → bench → ledger row
  harnessd/        The one door the agent talks through (7 JSON ops)
 agent/             Untrusted proposer (Claude Agent SDK, Python)
 playbook/          Optimization classes 1–7, tried strictly cheapest-first
 config/            accept.toml (thresholds), pricing.toml (ROI inputs)
-targets/           Per-target checkouts — the only agent-writable path
+targets/           Vendored OSS targets — the only agent-writable path
 corpora/           Hash-pinned golden-replay inputs (read-only to agent)
-results/           Calibration evidence, per-engagement outputs
+results/           Calibration evidence, case studies, generated ROI reports
 docs/              The mdBook site (deployed to GitHub Pages)
 ```
 ## Quick start
 ```sh
-cargo test --workspace          # trust-layer unit tests
+just build / test / lint       # trust-layer workspace
-just aa                         # A/A self-test: must yield a null verdict
+just aa                        # A/A self-test — must yield a null verdict
-just compare "cmd-a" "cmd-b"    # interleaved A/B with bootstrap CI
+just gates <target>            # corpus pin + upstream tests + golden replay
 just verdict <t> <bin> ...     # gates + bench vs pristine baseline + ledger row
 just report <run-id>           # ROI report from a ledger row
 just isolation-check           # 19 OS-boundary checks (both modes)
 ```
-## Status
+Full command reference and reproduction steps:
 [Reproduce it yourself](https://bdbrown4.github.io/peltier.io/reproduce.html).
-**Phase 0 complete** (see SPEC §5). Five attempts in the ledger across
+## How it stays honest
 two pinned targets (comrak 0.53.0, tokei 14.0.0), A/A-calibrated,
 every number with CI and workload:
- **tokei +10.4% median [95% CI +8.5%, +12.0%]** — read-path buffer
+- The agent speaks to the trust layer through **seven read-only-plus-one-
-  zeroing eliminated + fat LTO, byte-identical, 4,332-iteration
+  write JSON operations**; it has no shell, cannot write outside a target
-  diff-fuzz clean, sanitizers clean, auto-accepted
+  workspace, and cannot touch the ledger or thresholds.
-  (`results/phase0/case-study-tokei.md`). Meets the Phase 0 exit bar.
+- The **baseline is rebuilt from a pristine checkout** every session — a
- comrak +4.6% [+3.3%, +5.8%] — mimalloc swap, `needs-human-review`
+  patch never becomes its own comparison.
-  on an LSan reachability flag (`results/phase0/case-study-comrak.md`).
+- The **ledger is append-only** (SQLite triggers refuse UPDATE/DELETE);
- Two null/sub-threshold results correctly rejected and logged.
+  accepted rows are written only by the `verdict` binary after the gates.
 - The agent process runs under **OS-level isolation** (read-only mount
  namespace or an unprivileged uid) — the boundary is filesystem
  permissions, not a prompt.
-Next: Phase 1 trust-layer hardening (containers, perf, automated
+See [the case studies](https://bdbrown4.github.io/peltier.io/case-studies/overview.html)
-calibration, gate orchestration, agent IPC).
+— especially [the caught false-accept](https://bdbrown4.github.io/peltier.io/case-studies/comrak-false-accept.html),
-
+where the pipeline over-accepted a leaking patch and the audit overturned
-Done:
+it before it shipped.
 - Workspace with the four trust-layer crates; core statistics
  (interleaved scheduling, bootstrap ratio CIs, CI-lower-bound accept
  rule), append-only ledger schema with mutation-refusing triggers,
  corpus hash-pin verifier, equivalence-policy parser, ROI formulas —
  all unit-tested.
 - bench-runner CLI: whole-program `compare` and `aa` modes with env
  fingerprinting.
 - Playbook classes 1–7; agent tool contract and prompting spine.
 Next (Phase 1 exit criteria):
 - `perf stat` counters + max RSS + RAPL capture per run.
 - Automated A/A calibration sessions and regression-injection self-test
  recorded to `results/calibration/` (<5% FP, ≥95% detection).
 - diff-test orchestration end-to-end on two real targets; harness IPC
  for the agent tool surface.
 ## License
@@ -189,6 +189,15 @@ fn one_request(port: u16) -> std::io::Result<()> {
 /// One open-loop load session against a freshly spawned server.
 pub fn run_session(cfg: &ServiceCfg, port: u16) -> Result<SessionResult> {
    ensure!(
        cfg.rate.is_finite() && cfg.rate > 0.0,
        "arrival rate must be finite and positive (got {})",
        cfg.rate
    );
    ensure!(
        cfg.count > 0 && cfg.workers > 0,
        "count and workers must be > 0"
    );
    let mut child = spawn_server(cfg, port)?;
    // Warm up closed-loop (untimed) so the first timed request doesn't eat
    // page faults / cold caches.
@@ -0,0 +1,39 @@
 //! Apply a target's equivalence policy to two output files (SPEC §13
 //! kernel lane). Loads `targets/<target>/equivalence.toml` (byte-identical
 //! by default, or fp-tolerance) and reports whether the candidate output
 //! is equivalent to the baseline under it.
 //!
 //!   fp-compare <target-dir> <baseline-file> <candidate-file>
 use anyhow::{Context, Result};
 use diff_test::policy::EquivalencePolicy;
 use std::path::PathBuf;
 fn main() -> Result<()> {
    let mut args = std::env::args().skip(1);
    let target_dir = PathBuf::from(
        args.next()
            .context("usage: fp-compare <target-dir> <baseline> <candidate>")?,
    );
    let baseline_path = args.next().context("missing baseline file")?;
    let candidate_path = args.next().context("missing candidate file")?;
    let policy = EquivalencePolicy::load(&target_dir)?;
    let baseline = std::fs::read_to_string(&baseline_path)?;
    let candidate = std::fs::read_to_string(&candidate_path)?;
    println!("policy: {policy:?}");
    match policy.compare(&baseline, &candidate) {
        Ok(()) => {
            println!("EQUIVALENT under policy");
            Ok(())
        }
        Err(d) => {
            println!(
                "DIVERGENT at token/offset {}: baseline={:?} candidate={:?} — {}",
                d.token, d.baseline, d.candidate, d.reason
            );
            std::process::exit(1);
        }
    }
 }
@@ -40,6 +40,96 @@ impl EquivalencePolicy {
        }
        Ok(policy)
    }
    /// Compare two outputs under this policy. `ByteIdentical` requires an
    /// exact match. `FpTolerance` tokenizes on whitespace and compares
    /// numeric tokens within `abs + rel*|baseline|`; non-numeric tokens
    /// (and mismatched token counts) must still match exactly. This is the
    /// gate the kernel lane (SPEC §13) needs: an optimization that reorders
    /// floating-point accumulation changes the last ULPs, so it fails
    /// byte-identical but passes a declared tolerance — while a genuine
    /// correctness bug (a wrong result) still exceeds it.
    pub fn compare(&self, baseline: &str, candidate: &str) -> Result<(), Divergence> {
        match self {
            Self::ByteIdentical => {
                if baseline == candidate {
                    Ok(())
                } else {
                    let pos = baseline
                        .bytes()
                        .zip(candidate.bytes())
                        .position(|(a, b)| a != b)
                        .unwrap_or(baseline.len().min(candidate.len()));
                    Err(Divergence {
                        token: pos,
                        baseline: snippet(baseline, pos),
                        candidate: snippet(candidate, pos),
                        reason: "byte-identical mismatch".into(),
                    })
                }
            }
            Self::FpTolerance { abs, rel } => {
                let bt: Vec<&str> = baseline.split_whitespace().collect();
                let ct: Vec<&str> = candidate.split_whitespace().collect();
                if bt.len() != ct.len() {
                    return Err(Divergence {
                        token: bt.len().min(ct.len()),
                        baseline: format!("{} tokens", bt.len()),
                        candidate: format!("{} tokens", ct.len()),
                        reason: "token-count mismatch".into(),
                    });
                }
                for (i, (b, c)) in bt.iter().zip(&ct).enumerate() {
                    match (b.parse::<f64>(), c.parse::<f64>()) {
                        (Ok(bn), Ok(cn)) => {
                            if bn.is_nan() && cn.is_nan() {
                                continue;
                            }
                            let d = (bn - cn).abs();
                            let tol = abs + rel * bn.abs();
                            // Explicit NaN handling: a lone NaN (the other
                            // side finite) is a divergence, not a pass.
                            if d.is_nan() || d > tol {
                                return Err(Divergence {
                                    token: i,
                                    baseline: (*b).to_string(),
                                    candidate: (*c).to_string(),
                                    reason: format!("|Δ|={d:.3e} exceeds tolerance {tol:.3e}"),
                                });
                            }
                        }
                        _ => {
                            if b != c {
                                return Err(Divergence {
                                    token: i,
                                    baseline: (*b).to_string(),
                                    candidate: (*c).to_string(),
                                    reason: "non-numeric token mismatch".into(),
                                });
                            }
                        }
                    }
                }
                Ok(())
            }
        }
    }
 }
 /// The first place two outputs diverge under a policy.
 #[derive(Debug, Clone, PartialEq)]
 pub struct Divergence {
    /// Token index (FpTolerance) or byte offset (ByteIdentical).
    pub token: usize,
    pub baseline: String,
    pub candidate: String,
    pub reason: String,
 }
 fn snippet(s: &str, pos: usize) -> String {
    let start = pos.saturating_sub(8);
    let end = (pos + 8).min(s.len());
    s.get(start..end).unwrap_or("").replace('\n', "\\n")
 }
 #[cfg(test)]
@@ -73,4 +163,28 @@ mod tests {
            }
        );
    }
    #[test]
    fn byte_identical_rejects_reordered_fp() {
        // The blocked kernel's low-bit difference: byte-identical says no.
        let p = EquivalencePolicy::ByteIdentical;
        assert!(p.compare("1.0000000 2.0", "1.0000001 2.0").is_err());
    }
    #[test]
    fn fp_tolerance_accepts_low_bits_rejects_real_error() {
        let p = EquivalencePolicy::FpTolerance {
            abs: 1e-6,
            rel: 1e-5,
        };
        // Last-ULP reordering: within tolerance.
        assert!(p.compare("1.0000000 42.0", "1.0000001 42.0").is_ok());
        // A genuine wrong result: exceeds tolerance, caught.
        let d = p.compare("1.0 42.0", "1.0 42.5").unwrap_err();
        assert_eq!(d.token, 1);
        // Non-numeric tokens still must match exactly.
        assert!(p.compare("ok 1.0", "bad 1.0").is_err());
        // Token-count mismatch is a divergence.
        assert!(p.compare("1.0 2.0", "1.0").is_err());
    }
 }
@@ -190,6 +190,28 @@ impl Ledger {
    /// onto a completed attempt (harnessd `read_verdict`). Returns the
    /// verdict plus the bench CIs; never enough to game, since the row
    /// is already written and immutable.
    /// Every attempt's (playbook_class, target, verdict, sanitizers_clean)
    /// — the training signal for the learned class-selection policy
    /// (SPEC §13). A "shippable win" is `accepted` AND sanitizers-clean,
    /// matching the report generator's definition.
    pub fn all_outcomes(&self) -> Result<Vec<(u8, String, String, bool)>, LedgerError> {
        let mut stmt = self
            .conn
            .prepare("SELECT playbook_class, target, verdict, gates FROM attempts")?;
        let rows = stmt.query_map([], |r| {
            let class: u8 = r.get(0)?;
            let target: String = r.get(1)?;
            let verdict: String = r.get(2)?;
            let gates: Option<String> = r.get(3)?;
            let sanitizers_clean = gates
                .and_then(|g| serde_json::from_str::<serde_json::Value>(&g).ok())
                .and_then(|g| g.get("sanitizers_clean").and_then(|v| v.as_bool()))
                .unwrap_or(false);
            Ok((class, target, verdict, sanitizers_clean))
        })?;
        Ok(rows.collect::<Result<Vec<_>, _>>()?)
    }
    /// Full row data the ROI report generator needs (SPEC §9). Includes
    /// the workload string pulled out of the bench evidence.
    pub fn report_row(&self, run_id: &str) -> Result<Option<serde_json::Value>, LedgerError> {
@@ -217,12 +239,30 @@ impl Ledger {
                    .and_then(|b| b.get("env_fingerprint"))
                    .and_then(|f| f.get("workload"))
                    .cloned();
                // Pull the sanitizer gate out so the report can refuse to
                // mint a clean ROI for an `accepted` row that predates the
                // machine-enforced sanitizer gate and was overturned by
                // audit (phase2-comrak-010). The gates JSON is stored
                // separately; read it here.
                let sanitizers_clean = self
                    .conn
                    .query_row(
                        "SELECT gates FROM attempts WHERE run_id = ?1",
                        [run_id],
                        |r| r.get::<_, Option<String>>(0),
                    )
                    .ok()
                    .flatten()
                    .and_then(|g| serde_json::from_str::<serde_json::Value>(&g).ok())
                    .and_then(|g| g.get("sanitizers_clean").and_then(|v| v.as_bool()))
                    .unwrap_or(false);
                Ok(Some(serde_json::json!({
                    "run_id": run_id,
                    "target": target,
                    "playbook_class": class,
                    "hypothesis": hypothesis,
                    "verdict": verdict,
                    "sanitizers_clean": sanitizers_clean,
                    "timestamp": ts,
                    "speedup_median": bench.as_ref().and_then(|b| b.get("speedup_median").cloned()),
                    "speedup_ci": bench.as_ref().and_then(|b| b.get("speedup_ci").cloned()),
@@ -0,0 +1,12 @@
 [package]
 name = "policy"
 description = "Learned class-selection policy (SPEC §13): rank playbook classes by the Wilson lower bound of their observed win-rate in the ledger, degrading to cheapest-first when data is thin"
 version.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
 [dependencies]
 anyhow.workspace = true
 clap.workspace = true
 ledger = { path = "../ledger" }
@@ -0,0 +1,162 @@
 //! Learned class-selection policy (SPEC §13 research fork).
 //!
 //! The append-only ledger is a growing dataset of `(playbook_class,
 //! target) -> verdict`. This turns that history into a *prior*: given a
 //! target, rank the playbook classes by how likely each is to yield a
 //! shippable win — learned from evidence, not the fixed cheapest-first
 //! order alone.
 //!
 //! The ranking statistic is the **Wilson score lower bound** of the
 //! win-rate, not the point estimate. This is the same discipline the rest
 //! of the project uses (quote the CI lower bound, never the median): a
 //! class that won 1/1 does NOT outrank one that won 8/10, because its
 //! interval is wide. With no data a class scores 0 and falls back to the
 //! cheapest-first prior. As the ledger grows, the estimates sharpen — the
 //! policy is honest about its own uncertainty by construction.
 /// Wilson score interval lower bound for `wins`/`n` at z (1.96 = 95%).
 /// Returns 0.0 for n == 0 (no evidence).
 pub fn wilson_lower_bound(wins: u64, n: u64, z: f64) -> f64 {
    if n == 0 {
        return 0.0;
    }
    let n = n as f64;
    let p = wins as f64 / n;
    let z2 = z * z;
    let denom = 1.0 + z2 / n;
    let center = p + z2 / (2.0 * n);
    let margin = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt();
    ((center - margin) / denom).max(0.0)
 }
 /// Aggregated evidence for one playbook class.
 #[derive(Debug, Clone, PartialEq)]
 pub struct ClassStat {
    pub class: u8,
    pub attempts: u64,
    /// Shippable wins: `accepted` AND sanitizers-clean.
    pub wins: u64,
    /// Real wins held for a human ruling (needs-human-review) — counted
    /// separately, since they are neither clean auto-wins nor failures.
    pub held: u64,
    pub win_rate: f64,
    pub wilson_lb: f64,
 }
 impl ClassStat {
    /// A short human note on the evidence state.
    pub fn note(&self) -> &'static str {
        if self.attempts == 0 {
            "untried — cheapest-first prior only"
        } else if self.wins > 0 {
            "proven winner"
        } else if self.held > 0 {
            "held for human review (measured win, tier-gated)"
        } else {
            "evidence against — tried, no shippable win"
        }
    }
 }
 /// Build per-class stats (classes 1..=7) from ledger outcome rows,
 /// optionally filtered to one target. Each outcome is
 /// `(class, target, verdict, sanitizers_clean)`.
 pub fn class_stats(
    outcomes: &[(u8, String, String, bool)],
    target: Option<&str>,
    z: f64,
 ) -> Vec<ClassStat> {
    (1u8..=7)
        .map(|class| {
            let rows = outcomes
                .iter()
                .filter(|(c, t, _, _)| *c == class && target.is_none_or(|want| t == want));
            let (mut attempts, mut wins, mut held) = (0u64, 0u64, 0u64);
            for (_, _, verdict, san) in rows {
                attempts += 1;
                if verdict == "accepted" && *san {
                    wins += 1;
                } else if verdict == "needs-human-review" {
                    held += 1;
                }
            }
            let win_rate = if attempts > 0 {
                wins as f64 / attempts as f64
            } else {
                0.0
            };
            ClassStat {
                class,
                attempts,
                wins,
                held,
                win_rate,
                wilson_lb: wilson_lower_bound(wins, attempts, z),
            }
        })
        .collect()
 }
 /// Rank classes for selection. Primary key: Wilson lower bound (evidence
 /// of success), descending. Ties (notably all the zero-evidence classes)
 /// break by: fewer failed attempts first (prefer untried over
 /// proven-failure), then cheaper class first (the cost prior).
 pub fn rank(mut stats: Vec<ClassStat>) -> Vec<ClassStat> {
    stats.sort_by(|a, b| {
        b.wilson_lb
            .partial_cmp(&a.wilson_lb)
            .unwrap_or(std::cmp::Ordering::Equal)
            .then_with(|| {
                // failures = attempts without a win or a hold
                let fa = a.attempts - a.wins - a.held;
                let fb = b.attempts - b.wins - b.held;
                fa.cmp(&fb)
            })
            .then_with(|| a.class.cmp(&b.class))
    });
    stats
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[test]
    fn wilson_is_zero_without_evidence() {
        assert_eq!(wilson_lower_bound(0, 0, 1.96), 0.0);
    }
    #[test]
    fn wilson_penalizes_small_samples() {
        // 1/1 must NOT outrank 8/10 — the small sample's interval is wide.
        let one_of_one = wilson_lower_bound(1, 1, 1.96);
        let eight_of_ten = wilson_lower_bound(8, 10, 1.96);
        assert!(
            one_of_one < eight_of_ten,
            "1/1 ({one_of_one:.3}) should rank below 8/10 ({eight_of_ten:.3})"
        );
    }
    #[test]
    fn wilson_monotonic_in_evidence() {
        // Same rate, more evidence → higher lower bound.
        assert!(wilson_lower_bound(50, 100, 1.96) > wilson_lower_bound(5, 10, 1.96));
    }
    #[test]
    fn rank_prefers_proven_then_untried_then_failed() {
        let outcomes = vec![
            (5u8, "t".into(), "accepted".into(), true),
            (5u8, "t".into(), "accepted".into(), true),
            (5u8, "t".into(), "rejected-bench".into(), false),
            (6u8, "t".into(), "rejected-bench".into(), false), // evidence against
                                                               // class 7 untried
        ];
        let ranked = rank(class_stats(&outcomes, None, 1.96));
        assert_eq!(ranked[0].class, 5, "proven class 5 ranks first");
        // untried class (7) must rank above the tried-and-failed class 6
        let pos7 = ranked.iter().position(|s| s.class == 7).unwrap();
        let pos6 = ranked.iter().position(|s| s.class == 6).unwrap();
        assert!(pos7 < pos6, "untried class 7 outranks failed class 6");
    }
 }
@@ -0,0 +1,104 @@
 //! `policy rank` — the learned class-selection prior (SPEC §13).
 //! Reads the append-only ledger and ranks playbook classes by the Wilson
 //! lower bound of their observed shippable-win rate, degrading to
 //! cheapest-first where evidence is thin. Advisory only: the gates still
 //! decide every verdict. This just tells the agent where to look first.
 use anyhow::Result;
 use clap::Parser;
 use policy::{class_stats, rank};
 use std::path::PathBuf;
 const CLASS_NAME: [&str; 8] = [
    "",
    "build-config",
    "allocator",
    "alloc-churn",
    "data-layout",
    "algorithmic",
    "simd",
    "concurrency",
 ];
 #[derive(Parser)]
 #[command(
    name = "policy",
    about = "Learned playbook-class ranking from the ledger (SPEC §13)"
 )]
 struct Cli {
    /// Restrict the evidence to one target (else pool all targets).
    #[arg(long)]
    target: Option<String>,
    #[arg(long, default_value = "results/ledger.sqlite")]
    db: PathBuf,
    /// z for the Wilson interval (1.96 = 95%).
    #[arg(long, default_value_t = 1.96)]
    z: f64,
 }
 fn main() -> Result<()> {
    let cli = Cli::parse();
    let ledger = ledger::Ledger::open(&cli.db)?;
    let outcomes = ledger.all_outcomes()?;
    let ranked = rank(class_stats(&outcomes, cli.target.as_deref(), cli.z));
    let scope = cli.target.as_deref().unwrap_or("all targets");
    println!(
        "learned class-selection ranking — evidence: {} ledger rows, scope: {scope}\n\
         (Wilson 95% lower bound of the shippable-win rate; ties break untried-before-failed, then cheapest-first)\n",
        outcomes.len()
    );
    println!(
        "{:>4}  {:<13} {:>8} {:>5} {:>5} {:>9} {:>10}  note",
        "rank", "class", "attempts", "wins", "held", "win-rate", "wilson-lb"
    );
    println!("{}", "-".repeat(78));
    for (i, s) in ranked.iter().enumerate() {
        println!(
            "{:>4}  {:>1} {:<11} {:>8} {:>5} {:>5} {:>8.0}% {:>10.3}  {}",
            i + 1,
            s.class,
            CLASS_NAME[s.class as usize],
            s.attempts,
            s.wins,
            s.held,
            s.win_rate * 100.0,
            s.wilson_lb,
            s.note(),
        );
    }
    // The actionable line: the first class with real supporting evidence,
    // else the cheapest untried class.
    let best_evidenced = ranked.iter().find(|s| s.wilson_lb > 0.0);
    let cheapest_untried = ranked
        .iter()
        .filter(|s| s.attempts == 0)
        .min_by_key(|s| s.class);
    println!();
    match (best_evidenced, cheapest_untried) {
        (Some(e), _) => println!(
            "recommendation: try class {} ({}) first — best evidence (Wilson lb {:.3}); \
             then the next-ranked class whose profile preconditions match.",
            e.class, CLASS_NAME[e.class as usize], e.wilson_lb
        ),
        (None, Some(u)) => println!(
            "recommendation: no class has supporting evidence yet; fall back to the \
             cheapest untried class {} ({}).",
            u.class, CLASS_NAME[u.class as usize]
        ),
        (None, None) => {
            println!("recommendation: no evidence and no untried class; use cheapest-first.")
        }
    }
    println!(
        "\nAdvisory only — the equivalence + significance gates decide every verdict. \
         This ranking sharpens as the ledger grows."
    );
    println!(
        "note: a 'win' counts only machine-sanitizer-verified accepts. Pre-sanitizer-gate \
         accepts and the overturned comrak-010 are conservatively excluded — the policy \
         trusts the ledger's machine record, not the external audit narrative."
    );
    Ok(())
 }
@@ -69,14 +69,27 @@ fn main() -> Result<()> {
        "**Target:** {target} · **Playbook class:** {class} · **Verdict:** `{verdict}`\n\n\
         **Workload:** {workload}\n\n",
    ));
-    // A report for anything the pipeline did not accept must say so before
+    // A report for anything the pipeline did not accept — OR an `accepted`
-    // any dollar figure — the ROI below is a measured effect, not a
+    // row that is not sanitizer-clean (a pre-sanitizer-gate accept the
-    // shippable saving.
+    // audit overturned, e.g. phase2-comrak-010) — must say so before any
-    if verdict != "accepted" {
+    // dollar figure. The ROI below is a measured effect, not a shippable
    // saving. This closes the hole where the immutable ledger's historical
    // accept could mint a clean ROI for a patch that was never shipped.
    let sanitizers_clean = row["sanitizers_clean"].as_bool().unwrap_or(false);
    let shippable = verdict == "accepted" && sanitizers_clean;
    if !shippable {
        let reason = if verdict != "accepted" {
            format!("its verdict is `{verdict}`, not `accepted`")
        } else {
            "it is `accepted` but NOT sanitizer-clean — this is a pre-sanitizer-gate accept that \
             the audit overturned (the immutable ledger preserves the original verdict; \
             `sanitizers_clean:false` is the disqualifying signal)"
                .to_string()
        };
        md.push_str(&format!(
-            "> ⚠️ **Not shipped.** This attempt's verdict is `{verdict}`, not `accepted`. Any ROI\n\
+            "> ⚠️ **Not shipped.** This attempt is not a committed saving because {reason}. Any ROI\n\
-             > below is the *measured* effect — it did not clear the significance/tier bar and is\n\
+             > below is the *measured* effect only; it did not clear the significance/tier/sanitizer\n\
-             > **not** a committed saving. Shown for completeness; the ledger records it as-is.\n\n",
+             > bar. Shown for completeness; the ledger records it as-is.\n\n",
        ));
    }
@@ -0,0 +1,20 @@
 [book]
 title = "hotpath"
 description = "A profile-guided optimization agent that only ships wins it can prove."
 authors = ["hotpath"]
 language = "en"
 src = "src"
 [output.html]
 default-theme = "ayu"
 preferred-dark-theme = "ayu"
 git-repository-url = "https://github.com/bdbrown4/peltier.io"
 edit-url-template = "https://github.com/bdbrown4/peltier.io/edit/main/docs/{path}"
 additional-css = ["theme/custom.css"]
 [output.html.fold]
 enable = true
 level = 1
 [output.html.playground]
 runnable = false
@@ -0,0 +1,28 @@
 # Summary
 [hotpath](./introduction.md)
 - [The thesis: trust is the product](./thesis.md)
 - [Architecture](./architecture.md)
 - [Measurement discipline](./measurement.md)
 - [Equivalence gates](./equivalence.md)
 - [The agent loop](./agent-loop.md)
 - [Isolation & anti-reward-hacking](./isolation.md)
 - [The playbook](./playbook.md)
 - [ROI: from stopwatch to dollars](./roi.md)
 # Case studies
 - [Overview](./case-studies/overview.md)
 - [tokei — compounding algorithmic wins (Rust)](./case-studies/tokei.md)
 - [Cheap wins first: build flags & allocators](./case-studies/build-flags.md)
 - [The caught false-accept (comrak)](./case-studies/comrak-false-accept.md)
 - [cJSON — the cross-language proof (C)](./case-studies/cjson.md)
 - [Services & scale: latency under load](./case-studies/service.md)
 - [Research forks: kernel lane & learned policy](./research-forks.md)
 # Reference
 - [Reproduce it yourself](./reproduce.md)
 - [The trust boundary (infra)](./infra.md)
 - [Roadmap & scope](./roadmap.md)
@@ -0,0 +1,52 @@
 # The agent loop
 The proposer is a Claude Agent SDK loop (`agent/hotpath_agent/`) driving
 the seven harness tools over a standalone MCP server. It has no built-in
 file, shell, or web tools in its allowlist — only the harness door.
 ## One attempt, start to finish
 1. **Read the ledger.** What classes and hypotheses were already tried on
   this target? The ledger is the anti-double-attempt memory: a class may
   be re-entered with a *materially new* hypothesis, never a duplicate.
 2. **Read the profile.** callgrind cache-sim ranks hotspots with per-
   function cache-miss and branch-mispredict columns, so the agent can tell
   an allocation-churn hotspot from a branch-bound one and pick the right
   playbook class.
 3. **Pick the cheapest untried class** whose profile preconditions are met,
   and read its playbook entry.
 4. **Read the source** (windowed, at the pinned commit) and **propose a
   patch** with a stated hypothesis.
 5. **Run the verdict** and **poll** for the result.
 6. **Report it verbatim.** A rejection with a clean ledger row is a
   *successful* outcome. Do not iterate more than twice on a rejected
   hypothesis without a new idea.
 ## What the loop proved about itself
 Across an unattended grind of 20+ audited attempts, the loop demonstrated
 the behavior the architecture is designed to produce:
 - It chose the cheapest untried class each time, reading the ledger to
  avoid re-grinding a dead end.
 - Every patch went through `propose_patch` (path-allowlist + `git apply`)
  and the full gate+bench pipeline. Malformed diffs were refused by the
  harness, not the agent's goodwill.
 - **It twice refused to fabricate a verdict it could not read back.** When
  a multi-minute pipeline outran the MCP transport's timeout, the agent
  checked the ledger, confirmed the row wasn't written, and reported the
  verdict as *unreadable, not invented* — asking a human to read the
  ledger rather than guessing a number. That refusal, under a real failure
  mode, is the anti-reward-hacking design working.
 ## The honest-failure record
 The ledger records **every** attempt, including the rejections, because a
 rejection is training data and a demonstration of the bar. Of the audited
 unattended attempts, most were honest rejections — real mechanisms that
 simply didn't clear the 2% CI-lower-bound bar — a handful were verified
 wins, and one was a `rejected-gate` where golden replay caught the agent's
 patch producing wrong output that a green test suite would have shipped.
 The load-bearing number is **zero shipped false accepts** across the whole
 project.
@@ -0,0 +1,61 @@
 # Architecture
 hotpath is split into a **trust layer** (Rust, the agent cannot write it)
 and an **untrusted proposer** (the agent). The split is structural, not
 advisory: the guarantees hold even if the proposer is adversarial.
 ```
 crates/            Trust layer — agent has NO write access (SPEC §10)
  bench-runner/    Interleaved A/B timing, bootstrap CIs, A/A calibration,
                   service-mode latency (coordinated-omission correct)
  diff-test/       Equivalence gates, corpus hash-pinning, per-target spec
  ledger/          Append-only SQLite attempt ledger (enforced by triggers)
  report/          ROI: speedup CI → cores → dollars, methodology inline
  verdict/         The pipeline in one command: gates → bench → ledger row
  harnessd/        The one door the agent talks through (7 JSON ops)
 agent/             Untrusted proposer (Claude Agent SDK, Python)
 playbook/          Optimization classes 1–7, tried strictly cheapest-first
 config/            accept.toml (thresholds), pricing.toml (ROI inputs)
 targets/           Vendored OSS targets — the only agent-writable path
 corpora/           Hash-pinned golden-replay inputs (read-only to agent)
 results/           Calibration evidence, case studies, generated reports
 ```
 ## The seven-tool boundary
 The agent never runs a shell, never writes a file, never touches the
 ledger directly. It speaks to `harnessd` — the only door — through exactly
 seven JSON operations:
 | Tool | What it can do |
 |---|---|
 | `read_profile` | Read a target's ranked hotspots (read-only) |
 | `read_ledger` | Read prior attempts — the anti-double-attempt memory |
 | `read_playbook` | Read an optimization class's preconditions/procedure |
 | `read_target_source` | Read a workspace file at the pinned commit (windowed) |
 | `propose_patch` | Submit a unified diff — path-allowlisted, then `git apply` |
 | `run_verdict` | Launch the gate+bench pipeline (detached) |
 | `read_verdict` | Poll the append-only ledger for the result |
 Note what is *absent*: no write outside `targets/<name>/workspace`, no
 shell, no way to run the bench or write the ledger except through the
 `verdict` binary, which only records a pass after the full gated pipeline.
 ## The data flow of one attempt
 ```
 profile ─▶ agent hypothesizes ─▶ propose_patch ─▶ [path allowlist + git apply]
                                                          │
   ledger row ◀─ verdict ◀─ bench (interleaved A/B) ◀─ gates (tests, golden,
   (append-only)             vs pristine-rebuilt         fuzz, sanitizers)
                             baseline
 ```
 The baseline is **rebuilt from a pristine checkout every session** — never
 from the agent's workspace — so a patch cannot poison its own comparison.
 The verdict is decided by the CI-lower-bound rule and written to an
 append-only ledger the agent cannot mutate.
 The next chapters walk each stage: [measurement](./measurement.md),
 [equivalence gates](./equivalence.md), [the agent loop](./agent-loop.md),
 and the [isolation](./isolation.md) that makes the boundary load-bearing.
@@ -0,0 +1 @@
 {{#include ../../../results/phase3/02-build-flags-and-allocators.md}}
@@ -0,0 +1 @@
 {{#include ../../../results/phase3/04-cjson-c-target.md}}
@@ -0,0 +1 @@
 {{#include ../../../results/phase3/03-comrak-caught-false-accept.md}}
@@ -0,0 +1,17 @@
 # Case studies
 Five verified case studies, each reproducible from a pinned corpus through
 the same gated pipeline, each number carrying its 95% CI and workload.
 - [tokei — compounding algorithmic wins (Rust)](./tokei.md) — three class-5
  wins on the same hot function, each unattended and audited.
 - [Cheap wins first: build flags & allocators](./build-flags.md) — LTO
  accepted (+10%), mimalloc held for a human ruling then accepted-scoped.
 - [The caught false-accept (comrak)](./comrak-false-accept.md) — the
  leaking teardown the bench loved, overturned by audit. The most important
  study.
 - [cJSON — the cross-language proof (C)](./cjson.md) — a sub-threshold win
  rejected, then a byte-identical win accepted, on a C target.
 - [Services & scale: latency under load](./service.md) — the cJSON win
  measured as a latency delta under coordinated-omission-correct replay,
  turned into a dollar figure mechanically.
@@ -0,0 +1 @@
 {{#include ../../../results/phase4/README.md}}
@@ -0,0 +1 @@
 {{#include ../../../results/phase3/01-tokei-algorithmic-rust.md}}
@@ -0,0 +1,52 @@
 # Equivalence gates
 A faster binary that behaves differently is not an optimization — it is a
 regression with good marketing. Before any speedup is even measured, the
 candidate must prove it does the same thing the baseline does. These are
 **hard gates**: a failure is `rejected-gate` and the bench never runs.
 ## The layers
 1. **Upstream test suite.** The target's own tests, run against the patched
   code. For cJSON that is the vendored Unity/CTest suite (19 tests:
   `parse_number`, `parse_string`, `print_*`, `misc_tests`, …); for the Rust
   targets, `cargo test`.
 2. **Golden replay — byte-identical.** The candidate processes the
   hash-pinned corpus and its output is hashed and compared to a pinned
   `GOLDEN.sha256`. A code counter that miscounts one line, a JSON printer
   that serializes one float differently — caught here. The corpus itself
   is pinned via `MANIFEST.sha256` and the harness refuses to run on a
   mismatch.
 3. **Differential fuzzing.** Pristine vs candidate on thousands of mutated
   inputs, comparing output. Every accepted win cleared **10,000 mutated
   inputs with 0 divergences** — the tokei wins after canonicalizing benign
   parallel-walker output ordering (a real finding: differential gates must
   compare *semantics*, not bytes, where the target is legitimately
   nondeterministic in presentation), the cJSON win on quote/number/escape
   edge cases.
 4. **Sanitizers.** ASan/LSan for Rust, ASan+UBSan for C. Run on **every
   would-be accept** — this is machine-enforced, not manual (see below).
 ## The tier rule (SPEC §8)
 Some changes can pass every gate and still be unsafe to auto-ship. A change
 that touches floating-point ordering, concurrency primitives, `unsafe`, or
 carries a sanitizer flag is **never** auto-accepted — its verdict is capped
 at `needs-human-review`. The tier is decided by the *nature* of the change,
 never by the size of the number. The mimalloc allocator swap
 ([case study](./case-studies/build-flags.md)) is the canonical example: a
 real +4.6% win, correctly held for a human ruling because a global
 allocator change's safety is not something one benchmark can clear.
 ## When a gate caught the pipeline itself
 The sanitizer gate exists in its current, machine-enforced form because it
 once *wasn't*. The pipeline auto-accepted a comrak patch that skipped AST
 teardown for a +10.7% win — and the audit's manual LeakSanitizer run found
 it leaks the arena at exit. It was overturned before shipping and the fix
 made `verdict` build with ASan/UBSan and run the pinned workload on every
 accept, capping any flagged win at `needs-human-review`. The full story is
 [the caught false-accept](./case-studies/comrak-false-accept.md).
@@ -0,0 +1 @@
 {{#include ../../docs/infra.md}}
@@ -0,0 +1,46 @@
 # hotpath
 > Spot cooling for hot paths. A profile-guided optimization agent that
 > only ships wins it can prove.
 hotpath profiles real binaries and services, isolates hot paths, proposes
 optimizations, and accepts a change **only** when two independent bars are
 cleared:
 1. **Behavioral equivalence** — the change provably preserves behavior
   (upstream tests, byte-identical golden replay, differential fuzzing,
   sanitizers) or it is discarded.
 2. **Statistical significance** — the speedup's bootstrap 95% CI lower
   bound clears a threshold on trusted, calibrated hardware.
 Everything else is commentary. A change without both is not a win; it is a
 ledger row explaining why not.
 ## What it has actually done
 Every number here carries its 95% confidence interval and its workload,
 and every one survived the gates above. These are not projections.
 | Result | Verified |
 |---|---|
 | tokei (Rust) — three compounding class-5 algorithmic wins | +10.7%, +3.7%, +9.9%, each CI-significant |
 | tokei — link-time optimization (class 1) | +10.4%, CI [+8.5%, +12.0%] |
 | cJSON (C) — number-handling rewrite (class 5) | +8.85%, CI [+7.5%, +10.0%] |
 | cJSON HTTP service — p50 latency under replayed load | +6.2%, CI [+5.8%, +7.2%] |
 | comrak (Rust) — mimalloc allocator swap (class 2) | +4.6%, CI [+3.3%, +5.8%], human-ruled |
 Across **five phases and 33 ledger rows**, with **zero shipped false
 accepts** — including two occasions where the pipeline itself over-accepted
 and the audit caught it before anything shipped, each becoming a permanent
 new gate.
 ## The differentiator
 Consultants sell surveys; compilers sell flags. hotpath sells **verified
 deltas with the methodology attached** — a stopwatch on pinned hardware,
 a proof of equivalence, and an ROI figure whose confidence interval you
 can quote to a hostile reviewer. That is the entire product, and this site
 is how it works.
 Start with [the thesis](./thesis.md), or jump to a
 [case study](./case-studies/overview.md).
@@ -0,0 +1,53 @@
 # Isolation & anti-reward-hacking
 The guarantees only matter if the proposer cannot cheat. hotpath's defense
 is **structural**: the agent is denied the capabilities it would need to
 manufacture a false win, at the OS level, not by prompt.
 ## What holds even if the proposer is adversarial
 - **Writes go through one door.** `propose_patch` is the only harness-
  mediated write path: a diff is path-allowlisted (relative paths only, no
  `..`, no `.git`, nothing outside `targets/<name>/workspace`) and then
  `git apply`-ed rooted in that workspace. A patch aimed at `config/` or
  `crates/` is refused before any apply. Verified against a malicious
  accept-threshold diff.
 - **The ledger is append-only.** `UPDATE` and `DELETE` are refused by
  SQLite triggers; `accepted` rows are written only by the `verdict`
  binary after the gated pipeline. The agent cannot forge a passing
  verdict through the harness.
 - **The baseline is unbypassable.** It is rebuilt from a pristine checkout;
  the agent's workspace never becomes the comparison.
 ## OS-level isolation (SPEC §10)
 The SDK's tool allow/deny list is defense-in-depth, not the boundary —
 `scripts/agent-isolated.sh` provides the real one, verified by
 `just isolation-check` (19 checks, both modes):
 - **`mountns` mode** — the whole agent process tree runs in a private mount
  namespace with the repo bind-mounted **read-only** and `CAP_SYS_ADMIN`/
  `CAP_SYS_PTRACE` dropped from the bounding set. Repo writes fail with
  `EROFS`; a forged `INSERT`, a direct `verdict` invocation, a `mount -o
  remount,rw`, and even userns re-entry to undo the mount are all refused.
 - **`user` mode** — the agent runs as an unprivileged user whose only write
  path into the repo is the harnessd Unix socket; the trust layer is owned
  by another uid. This is the production shape (`docs/infra.md`).
 ## The two times the pipeline was wrong
 Anti-reward-hacking is not "it never fails" — it is "when it fails, the
 audit catches it and the system gets harder." That happened twice, and
 both are documented, not hidden:
 1. A leaking teardown patch the bench loved was **auto-accepted** and then
   **overturned** by the 100% human audit's LeakSanitizer run. The fix made
   the sanitizer gate machine-enforced on every accept.
 2. During the [adversarial review](https://github.com/bdbrown4/peltier.io/blob/main/results/adversarial-review.md),
   the ROI report generator was found to mint a clean dollar figure for that
   same overturned row (it trusted the immutable ledger's historical
   `accepted` verdict). The fix made the report treat an accepted-but-not-
   sanitizer-clean row as **not shippable**.
 Both defects were found by attacking the system's own output, fixed, and
 turned into permanent gates. **Zero false accepts shipped.**
@@ -0,0 +1,64 @@
 # Measurement discipline
 The product is a trustworthy number, so the measurement is where most of
 the engineering went. The reference is Mytkowicz et al., *Producing Wrong
 Data Without Doing Anything Obviously Wrong* — the default state of a
 benchmark is "subtly lying."
 ## Interleaved A/B, never sequential
 Baseline and candidate run **interleaved** (ABABAB…), never in sequential
 blocks. Thermal drift, a background cron job, a noisy neighbor — anything
 that changes over the run hits both sides equally instead of biasing one.
 Warm-up runs are discarded.
 ## The baseline is always pristine
 The baseline binary is **rebuilt from a clean checkout** each session, with
 the candidate's patch stashed. A patch can never silently become part of
 its own baseline (a real bug that bit an early smoke test, now closed by
 `verdict --rebuild-baseline`).
 ## Bootstrap CIs, and the lower-bound rule
 Each side yields ≥30 measured samples. The statistic is the **bootstrap
 95% CI of the ratio-of-medians** (deterministic xorshift seed, so a result
 reproduces). A change is accepted only if the **CI lower bound** clears the
 threshold in `config/accept.toml` (default 2%) — not the median. The number
 you commit to is the one that survives the interval.
 ## The hardware has to earn trust first
 Before any measurement session, the harness runs two self-tests, recorded
 in `results/calibration/`:
 - **A/A**: run the *same* binary as both sides. It must produce a null
  verdict. A "speedup" from identical binaries is a calibration failure.
 - **Regression injection**: a synthetic 5% slowdown (a real busy-wait in
  the timed window) must be detected ≥95% of the time.
 CPU pinning (`taskset -c 2`) alone tightened the A/A confidence interval
 ~5× on the dev container. Every calibration anchor — local and on GitHub's
 shared runners — passed both bars before its numbers were trusted.
 ## Service mode: coordinated omission
 Phase 4 measures latency under load, where the classic trap is
 **coordinated omission**: a closed-loop client that waits for a slow
 response simply sends fewer requests during the stall, so the stall never
 appears in the latency histogram. hotpath's load generator is **open-loop**
 — requests are scheduled at a fixed rate and each latency is measured from
 its *intended* send time. When the server falls behind, the requests queued
 behind it each carry the full delay a real client would have seen. Nothing
 is omitted.
 The service measurement is calibrated the same way: 0/10 A/A false
 positives, 10/10 injected-5%-latency-regression detections
 (`results/calibration/cjson-service-aa.json`).
 > **Percentiles are claimed only when the measurement can pin them down.**
 > The cJSON service's p50 latency win was CI-tight and accepted (+6.2%,
 > [+5.8%, +7.2%]); its p99 CI was [0.07, 4.97] — uselessly wide on a
 > single-worker loopback tail — so it was **rejected**, even though the
 > median pointed the right way. A number the measurement can't pin down
 > does not ship.
@@ -0,0 +1,33 @@
 # The playbook
 Optimizations are tried in a fixed order — **cheapest and safest first** —
 because cost-per-attempt and risk both rise as you go down the list, and
 the ledger stops you from re-grinding a class that already has a verdict.
 | # | Class | Mechanism | Verified on |
 |---|---|---|---|
 | 1 | Build config | LTO, codegen-units, target-cpu | tokei ✅ +10.4% |
 | 2 | Allocator | Global allocator swap (mimalloc) | comrak ✅ +4.6% (human-ruled) |
 | 3 | Allocation churn | Remove per-item malloc/free | tokei ✅, cJSON ✅ (+ one overturned) |
 | 4 | Data layout | Make the cache hierarchy fit the access pattern | comrak (attempted) |
 | 5 | Algorithmic | Eliminate redundant work | tokei ×3 ✅, cJSON ✅ |
 | 6 | SIMD | Autovectorization enablement | comrak, tokei (attempted) |
 | 7 | Concurrency | Parallelism — always `needs-human-review` | — |
 Six classes were exercised across the case studies, with verified wins on
 classes 1, 2, 3, and 5 across **both a Rust and a C target**.
 ## Why the ordering is load-bearing
 The cheapest class produced the project's single largest individual win:
 LTO on tokei, **+10.4%**, a pure build-config change with no source edit.
 The lesson the ledger enforces is the one every performance engineer learns
 the hard way — *you do not hand-optimize a hot loop before you have turned
 on the optimizer's own switches.* An agent that jumped straight to SIMD
 would have missed a bigger, safer, free win sitting one config flag away.
 Each class entry (`playbook/0N-*.md`) carries its **preconditions** (the
 profile signature that justifies trying it), its **procedure**, its
 **verification** requirements, and its **known failure modes**. The agent
 reads the profile, matches the signature, and picks the cheapest class
 whose preconditions are met and that isn't already spent on this hotspot.
@@ -0,0 +1,50 @@
 # Reproduce it yourself
 Nothing here is a screenshot. Every number regenerates from a pinned corpus
 through the same commands.
 ## Prerequisites
 - Rust (stable) + `just`; `clang`/`cmake` for the C target; `valgrind` for
  callgrind profiling. Nightly Rust for the sanitizer gate.
 - Targets are vendored fresh (their `workspace/` is gitignored); each
  `targets/<name>/target.toml` has the `[fetch]` recipe.
 ## The commands
 ```
 just build / test / lint       # trust-layer workspace
 just aa [cmd]                  # A/A self-test — must be a null verdict
 just calibrate <cmd> <out>     # A/A false-positive + injection detection
 just gates <target>            # corpus pin + upstream tests + golden replay
 just verdict <t> <bin> ...     # gates + bench vs pristine baseline + ledger row
 just report <run-id>           # ROI report from a ledger row
 ```
 Service mode (Phase 4):
 ```
 just service <baseline-bin> <candidate-bin> <doc>     # interleaved p50/p99 CIs
 just service-calibrate <server> <doc> <out>           # latency A/A + injection
 just report <run-id> --service-json <json>            # latency + ROI report
 ```
 The agent loop, behind OS isolation:
 ```
 just isolation-check                    # 19 boundary checks, both modes
 scripts/agent-isolated.sh <target> <run-id>   # one unattended attempt, confined
 ```
 ## Auditing a result
 Every accepted win can be re-audited independently:
 ```
 python3 scripts/audit-attempt.py <run-id>        # mechanical re-check
 python3 scripts/diff-fuzz-<target>.py <dir> 10000 <candidate-bin>   # 10k fuzz
 ```
 The ledger itself is the record: `results/ledger.sqlite`, append-only,
 every attempt (win or loss) with its hypothesis, gates, bench evidence, and
 verdict.
@@ -0,0 +1,109 @@
 # Research forks: kernel lane & learned policy
 The core loop was complete at Phase 4 — profile → hypothesize → patch →
 gated-verdict → ROI, running unattended across Rust and C, batch and
 service, with zero shipped false accepts. SPEC §13 named two frontier ideas
 that the roadmap deliberately excluded, precisely because they change the
 *shape* of the work rather than extend the loop. Phase 5 built both, as
 stress tests with one question each: **does the trust machinery still hold?**
 Both times it does. Neither fork relaxes a non-negotiable; each routes its
 headline result to `needs-human-review`, and the learned prior is advisory
 only. What follows is the argument for each.
 ## The kernel lane: when byte-identical is the wrong gate
 Every target through Phase 4 was gated by **byte-identical golden replay** —
 the optimized output must match the baseline bit for bit. For a parser or an
 HTTP service that is exactly right. For a numerical kernel it is wrong, and
 the kernel lane is the case that proves it.
 `targets/matmul/kernel.c` is a single-precision matrix multiply in two forms:
 - **`matmul_ref`** — naive `i,j,k`, one sequential accumulator. The oracle.
 - **`matmul_opt`** — transpose B so the inner reduction walks contiguous
  memory (the cache win), plus eight independent accumulators tree-combined
  at the end (the ILP win, breaking the float-add dependency chain). The
  second lever **reorders the reduction**, so the last few ULPs differ *by
  construction*. Bit-identical output is impossible.
 The demonstration (`sh scripts/kernel-lane-demo.sh 512`) walks the argument:
 1. **byte-identical FAILs** — 244,901 of 262,144 values differ. Under the
   Phase 0–4 gate this correct 3× speedup would be *rejected*.
 2. **FP-tolerance is EQUIVALENT** — the target declares `mode =
   "fp-tolerance"`, `abs = 1e-4`, `rel = 1e-3`; every numeric token agrees
   within `abs + rel·|ref|`.
 3. **a wrong result is still caught** — perturbing one element by `+0.5` is
   REJECTED by the same tolerance. Loose enough for last-ULP reordering,
   tight enough for a real bug.
 4. **measured with the same machinery** — interleaved A/B, bootstrap 95% CI:
   **median 3.23×, CI [3.16, 3.26]**.
 The new rule lives in one place — `EquivalencePolicy::compare` in
 `diff-test`. Byte-identical stays exact; fp-tolerance tokenizes, compares
 numeric tokens within tolerance, handles NaN explicitly, and keeps
 non-numeric tokens and token counts exact. The `fp-compare` binary applies a
 target's `equivalence.toml` to two output files from the command line.
 The verdict is **`needs-human-review`**. Using the FP-tolerance tier *at
 all* is a human-review signal (SPEC §8): a person confirms the declared
 tolerance is defensible for the workload, exactly as with the mimalloc
 allocator swap. The machine measures and gates; the human ratifies the
 tolerance.
 **This is the GPU lane in miniature.** The environment has no GPU, so the
 GPU fork is shown on the CPU as what it actually is — the same trust
 machinery with a different timer. A Triton or CUDA kernel versus a reference
 kernel is the identical shape: reference-kernel differential testing within
 a tolerance, interleaved timing with a bootstrap CI, a ledger row routed to
 human review. Only the profiler and the bench clock become
 kernel-time-aware, and the hardware changes. The correctness story, the
 statistics, and the audit trail are already built and exercised here on real
 code.
 ## The learned policy: the ledger as a training set
 The playbook orders optimization classes cheapest-first — a fixed prior. But
 the ledger is now dozens of rows of `(class, target) → verdict`. That is a
 dataset, and it can turn the fixed ordering into a *learned* one.
 `crates/policy` ranks each class by the **Wilson score lower bound** of its
 observed shippable-win rate. The Wilson lower bound is the same instinct as
 the bootstrap CI lower bound the bench uses to accept a speedup: be
 pessimistic about thin evidence. A class that went 1-for-1 should not
 outrank one that went 3-for-16 — and it does not, because the lower bound
 penalizes small samples automatically. Ties break untried-before-failed (an
 unexplored class keeps its cheapest-first prior; a class with evidence
 *against* it sinks), then cheapest-first.
 Two guards keep it honest, both printed in the output footer:
 - A "win" counts **only** a machine-sanitizer-verified accept. The
  overturned `comrak-010` false-accept and every pre-sanitizer-gate accept
  are excluded — the policy trusts the ledger's machine record, not the
  audit narrative that caught them.
 - The tier-gated mimalloc win shows as *held for human review* — a third
  category, neither a loss nor a shippable win.
 On the current 34-row ledger the ranking recommends **algorithmic first**
 (best evidence, Wilson lb 0.066), then the next-ranked class whose profile
 preconditions match. It is advisory: the equivalence and significance gates
 still decide every verdict. This is the MLGO / AlphaDev / LLM-Compiler
 direction at the scale the ledger currently supports — a ranked prior that
 sharpens as the dataset grows and never overrides a gate.
 Run it with `cargo run -p policy` (add `--target <name>` to scope the
 evidence to one target).
 ## What the forks settle
 - A **learned prior** can be extracted from the ledger without weakening a
  single gate — it only reorders *what to try first*.
 - A **numerically-reordered optimization** — the class byte-identical replay
  cannot handle — slots into the same harness by swapping the equivalence
  policy, with the correctness bar preserved and the tolerance itself put
  under human review.
 The full write-up, with reproduction commands and the raw tool output, is in
 [`results/phase5/`](https://github.com/bdbrown4/peltier.io/tree/main/results/phase5).
@@ -0,0 +1,48 @@
 # Roadmap & scope
 ## Done — the core loop, end to end
 | Phase | Scope | Result |
 |---|---|---|
 | **0** | Manual dry run, two Rust targets | tokei +10.4% verified; methodology written |
 | **1** | Trust layer (bench-runner, diff-test, ledger) | A/A <5% FP, injection ≥95%, gates on 2 targets; calibrated free on GitHub runners |
 | **2** | Unattended agent loop | 20 audited attempts, 3 verified wins, **0 shipped false accepts**, OS-isolated |
 | **3** | Playbook + proof, C/C++ | cross-language cJSON win (+8.85%), 4 case studies, 6 playbook classes |
 | **4** | Services & scale | cJSON service +6.2% p50 under CO-correct replay, mechanical ROI report |
 | **5** | Research forks (SPEC §13) | FP-tolerance kernel lane (matmul 3.23×, needs-human-review) + learned class-selection prior from the ledger |
 The `profile → hypothesize → patch → gated-verdict → ROI` loop runs
 unattended, OS-isolated, across **Rust and C**, **batch and service**, from
 a pinned corpus to a dollar figure — with zero shipped false accepts across
 all phases, and the two pipeline over-accepts both caught by audit and
 turned into permanent gates.
 ## Phase 5 — the research forks (SPEC §13)
 Deliberately out of the core roadmap, these are the two hardest cases the
 spec named — both now built and exercised, both routed to human review.
 Full write-up: [Research forks](./research-forks.md).
 - **Kernel lane — done as a CPU demonstration.** A matmul optimization that
  reorders floating-point accumulation (transpose-B + eight-accumulator ILP)
  runs 3.23× faster and is *impossible* to gate byte-identically. It slots
  into the harness by swapping the equivalence policy for a declared
  FP-tolerance (`abs + rel·|ref|`) — which still catches a genuine wrong
  result — and routes to `needs-human-review` because using the tolerance
  tier at all is a §8 signal. This is the GPU lane in miniature:
  reference-kernel differential testing, interleaved timing, ledger row.
  Only the timer and the hardware change; **the actual GPU run needs GPU
  hardware**, absent here.
 - **Learned optimization policies — done.** `crates/policy` reads the ledger
  and ranks optimization classes by the Wilson lower bound of their
  shippable-win rate, turning the fixed cheapest-first ordering into a
  learned prior (advisory only — the gates still decide). Prior art: MLGO,
  AlphaDev, Meta's LLM Compiler.
 ## The real target
 The end goal is not vendored OSS case studies — it is a **real production
 system**. The same gates that matter for a JSON parser matter far more for
 a live CMS: an equivalence failure there is a customer-facing bug, and the
 no-network sandbox and append-only audit trail are exactly what make it
 safe to point an optimization agent at code that a business depends on.
@@ -0,0 +1,39 @@
 # ROI: from stopwatch to dollars
 The report is the pitch artifact, and it answers exactly one question:
 *what did the stopwatch say, and what does that buy?* It is generated
 **mechanically** — `just report <run-id>` reads a ledger row and the
 calibrated bench and renders the numbers; nothing is hand-typed.
 ## The three conversions
 - **Throughput → cores → dollars.** A speedup `s` on a CPU-bound fleet of
  `N` cores returns `N × (1 − 1/s)` cores; at a `$/core-hour` rate that is
  an annual saving. Both flow through the speedup's confidence interval, so
  the report quotes a *range*.
 - **Latency → percentile deltas → value.** Under replayed load, the p50/p99
  latency speedups with their CIs; a p50 service-time reduction maps to
  throughput capacity for a CPU-bound service.
 - **Batch windows → wall-clock hours** returned to a schedule.
 ## The rule that makes it survive review
 Every figure carries its CI and its workload, and the **methodology ships
 inside the report** — the interleaved A/B, the pristine baseline, the
 bootstrap CI, the gates, the calibration. And the number the report tells
 you to quote is the **CI lower bound**, never the median:
 > On a 500-core fleet at $0.04/core-hour, the cJSON service win (+6.2% p50
 > latency) returns **27.5 cores / $9,621 per year at the CI lower bound**
 > (median 29.4 / $10,290). The point estimate is not a promise.
 ## What the report refuses to do
 It will not mint a clean dollar figure for a change that wasn't shipped. A
 row whose verdict isn't `accepted`, **or** an `accepted` row that isn't
 sanitizer-clean (a pre-gate accept the audit overturned), gets a bold **Not
 shipped** banner before any number. That guard was itself added by the
 [adversarial review](https://github.com/bdbrown4/peltier.io/blob/main/results/adversarial-review.md),
 which caught the generator pitching the overturned comrak patch. The pitch
 artifact is held to the same standard as everything else: it cannot claim
 what wasn't proven.
@@ -0,0 +1,48 @@
 # The thesis: trust is the product
 Performance work has a credibility problem. "We made it 15% faster" is
 almost always some mix of a noisy benchmark, a workload that flatters the
 change, and a behavioral difference nobody checked. The number is real to
 the person who measured it and worthless to everyone downstream.
 hotpath is built on the opposite premise: **a performance claim is a
 liability until it is proven, and the proof ships with the number.**
 ## Three non-negotiables
 1. **No unverified performance claims.** A change without passing
   equivalence gates *and* a statistically significant benchmark delta
   does not exist. It is discarded and logged.
 2. **Measure before optimizing.** No patch is proposed until profile data
   identifies the hot path. A single benchmark run is never trusted.
 3. **Cheap wins first.** Build flags → LTO → PGO → allocator swap → *then*
   code changes. The ledger prevents re-attempting a dead end.
 ## What "proven" means here
 A win must clear **both** of two independent bars, because either one alone
 is gameable:
 - **Equivalence** stops "optimizations" that are actually behavior changes.
  A JSON parser that's faster because it stopped validating numbers is not
  faster; it's broken. Byte-identical golden replay and differential
  fuzzing catch this.
 - **Significance** stops noise from masquerading as a win. The bar is not
  "the median improved" — it is "the bootstrap 95% CI *lower bound* clears
  the threshold," measured by interleaving baseline and candidate on pinned
  hardware that has passed its own A/A calibration.
 A real +5.7% median that couldn't clear the significance bar (its CI lower
 bound was +0.2%) was **rejected** — and that rejection is in the ledger, because
 a rejection is a complete, valid outcome.
 ## Why an agent
 The loop — profile, hypothesize, patch, gate, measure, record — is
 mechanical and tedious, and the temptation to fudge a number is exactly
 what a tired human succumbs to. An agent proposes; the *trust layer*
 disposes. The agent cannot write the ledger, cannot move the baseline,
 cannot touch the thresholds. It can only propose a patch and ask the
 harness to judge it. The [architecture](./architecture.md) is designed so
 that even a compromised or adversarial proposer cannot manufacture a false
 win.
@@ -0,0 +1,20 @@
 /* hotpath docs — small refinements over the default mdBook theme. */
 .content h1 {
    border-bottom: 2px solid var(--links);
    padding-bottom: 0.2em;
 }
 /* Verified-number tables read as evidence: tighten and emphasize. */
 .content table td:first-child {
    font-weight: 500;
 }
 blockquote {
    border-inline-start: 4px solid var(--links);
 }
 /* Inline verdict-ish code (accepted / rejected-bench / needs-human-review) */
 .content code {
    font-variant-ligatures: none;
 }
@@ -84,3 +84,7 @@ service-calibrate server doc out *flags:
 # dollars and/or latency percentiles, CIs + methodology printed inline.
 report run_id *flags:
    cargo run -p report -- --run-id "{{run_id}}" {{flags}}
 # Learned playbook-class ranking from the ledger (SPEC §13 research fork)
 policy *flags:
    cargo run -p policy -- {{flags}}
@@ -0,0 +1,59 @@
 # Adversarial review — whole package
 A hostile pass over the entire project: every headline claim re-checked
 against the ledger, the trust boundary attacked directly, the newest
 code scrutinized hardest, and the statistics re-validated. Two real
 defects found and fixed; the load-bearing guarantees held.
 ## Findings
 ### 1. FIXED — the ROI generator minted a clean pitch for an overturned patch
 The report generator trusted the ledger's `verdict` field. `phase2-comrak-010`
 is immutably `accepted` (it predates the machine-enforced sanitizer gate;
 the audit overturned it for a LeakSanitizer leak and it was never banked).
 `report --run-id phase2-comrak-010` therefore produced a clean
 **$16,886/year** ROI report for a memory-leaking patch — exactly the kind
 of self-contradiction a hostile reviewer would seize on.
 **Fix:** the report now treats an `accepted` row that is not
 `sanitizers_clean` as **not shippable** and prints a "Not shipped" warning
 before any dollar figure. Because the pipeline now enforces ASan/UBSan on
 every accept, all legitimate wins are `sanitizers_clean: true`; only the
 historical overturned row is `accepted + false`, so the rule cleanly
 separates them. Regression-checked: `phase3-cjson-002` still renders clean.
 ### 2. FIXED — load generator could panic on degenerate inputs
 `bench-runner service --rate 0` (or `--count 0`) reached
 `Duration::from_secs_f64(1.0/0.0)` → panic. **Fix:** `run_session` now
 rejects a non-finite/non-positive rate and zero count/workers with a clear
 error.
 ## Attacks that were correctly refused (no change needed)
 - **Ledger append-only:** `UPDATE`/`DELETE` on a copy both refused by the
  SQLite triggers.
 - **Patch boundary:** `propose_patch` refused `../` traversal and `.git`
  access; `read_target_source` refused traversal. A patch can only touch
  `targets/<t>/workspace`.
 - **Accepted-row invariant:** all 10 `accepted` rows have green upstream
  tests + golden replay and a bootstrap CI lower bound ≥ 1.02.
 - **Statistics:** every batch and service A/A calibration is a null result
  (0 false positives); every injected-5% regression is detected (≥95%).
  The service load generator is coordinated-omission-correct and its
  injection test resolves a known 5% latency regression 10/10.
 ## Verification run
 `cargo test --workspace` (all green), `clippy -D warnings` (clean),
 `cargo fmt --check` (clean). Every case-study and CLAUDE.md figure
 re-checked against `results/ledger.sqlite` and the calibrated bench —
 consistent, with the compound tokei figure stated as an approximation
 (CIs don't multiply) and p99 service latency explicitly not claimed.
 ## Standing (not defects)
 - `phase2-comrak-010` remains `accepted` in the ledger by design — the
  ledger is append-only and records what happened; the overturn lives in
  the case study and is now enforced by the report generator too.
 - `phase0-comrak-002` (mimalloc) is `needs-human-review`, awaiting a ruling
  (see `results/rulings/`).
@@ -39,6 +39,14 @@ every workload, fragmentation pattern, and concurrency scenario the
 target will meet in production. The pipeline measured a real win and
 then **refused to auto-ship it**, tagging it for a human ruling instead.
 **Human ruling (`results/rulings/phase0-comrak-002.md`): ACCEPTED,
 scoped to the comrak CLI single-stream markdown-render workload** — a
 real win for the measured use, deliberately *not* a blanket "always use
 mimalloc" mandate. The tier forced the judgment; the judgment is bounded
 and written down. The ledger row stays `needs-human-review` (append-only
 records what the pipeline decided); the ruling is the human layer on
 top.
 ## What the pair demonstrates
 The cheap classes are not lesser — class 1 produced this project's
@@ -0,0 +1,139 @@
 # Phase 5 — the research forks (SPEC §13)
 Phase 5 exercises the two frontier ideas the spec deliberately left out of
 the core roadmap. Neither adds a new product; each is a stress test that
 answers one question: **does the trust machinery still hold when the shape
 of the work changes?** The answer, both times, is yes — the gates, the
 interleaved A/B timer, and the append-only ledger transfer unchanged; only
 the *gate policy* (5b) or the *ordering prior* (5a) is new.
 Nothing here relaxes a non-negotiable. Both forks route their headline
 result to `needs-human-review`, and the learned prior is advisory only —
 the gates still decide every verdict.
 ---
 ## 5a — learned class-selection policy
 **Artifact:** `crates/policy/`, output in `learned-policy.txt`.
 The ledger is now 34 rows of `(playbook class, target) → verdict`. That is
 a dataset. Phase 5a turns the *fixed* cheapest-first ordering of the
 playbook into a *learned* prior: rank each optimization class by the
 **Wilson score lower bound** of its observed shippable-win rate, and try the
 best-evidenced class first on a new hotspot.
 Why the Wilson lower bound and not the raw win rate: a class that went 1-for-1
 should not outrank a class that went 3-for-16. The Wilson lower bound
 penalizes small samples automatically — it is the same "be pessimistic
 about thin evidence" instinct as the bootstrap CI lower bound the bench
 uses to accept a speedup. Ties break *untried-before-failed* (an unexplored
 class keeps its cheapest-first prior; a class with evidence *against* it
 sinks), then cheapest-first.
 Two guards keep it honest, both visible in the output footer:
 - A "win" counts **only** a machine-sanitizer-verified accept. The
  overturned `comrak-010` false-accept and every pre-sanitizer-gate accept
  are excluded — the policy trusts the ledger's machine record, not the
  audit narrative that caught them.
 - The tier-gated allocator win (mimalloc) shows as `held for human review`,
  not as a loss and not as a shippable win — an accurate third category.
 Current ranking (all targets, 34 rows):
 | rank | class | attempts | wins | wilson-lb | note |
 |---|---|---|---|---|---|
 | 1 | algorithmic | 16 | 3 | 0.066 | proven winner |
 | 2 | alloc-churn | 4 | 1 | 0.046 | proven winner |
 | 3 | build-config | 9 | 1 | 0.020 | proven winner |
 | 4 | concurrency | 0 | 0 | 0.000 | untried — cheapest-first prior only |
 | 5 | allocator | 2 | 0 | 0.000 | held for human review (tier-gated) |
 | 6 | data-layout | 1 | 0 | 0.000 | evidence against |
 | 7 | simd | 2 | 0 | 0.000 | evidence against |
 The recommendation the agent would act on: **try algorithmic first** (best
 evidence), then the next-ranked class whose profile preconditions match.
 This is exactly the prior art direction (MLGO, AlphaDev, LLM Compiler) at
 the scale the ledger currently supports — a ranked prior, sharpening as the
 dataset grows, never overriding a gate.
 Reproduce: `cargo run -p policy` (add `--target cjson` to scope to one
 target).
 ---
 ## 5b — the kernel lane: FP-tolerance equivalence
 **Artifact:** `targets/matmul/`, `crates/diff-test/src/bin/fp-compare.rs`,
 demo in `kernel-lane.txt`.
 Every target through Phase 4 was gated by **byte-identical** golden replay:
 the optimized output must match the baseline bit for bit. That is the right
 gate for a parser or an HTTP service. It is the *wrong* gate for a numerical
 kernel, and the kernel lane is the case that proves it.
 `targets/matmul/kernel.c` is a single-precision matrix multiply in two
 implementations:
 - `matmul_ref` — naive `i,j,k` with one sequential accumulator. The oracle.
 - `matmul_opt` — transpose B (so the inner reduction walks contiguous
  memory: the cache win) **and** eight independent accumulators tree-combined
  at the end (breaking the float-add dependency chain: the ILP win). The
  second lever **reorders the reduction**, so the last few ULPs differ *by
  construction*. Bit-identical output is impossible.
 The demo (`sh scripts/kernel-lane-demo.sh 512`) walks the whole argument:
 1. **byte-identical FAILs** — 244,901 of 262,144 result values differ. Under
   the Phase 0–4 gate, this correct 3× speedup would be *rejected*.
 2. **the FP-tolerance policy is EQUIVALENT** — `equivalence.toml` declares
   `mode = "fp-tolerance"`, `abs = 1e-4`, `rel = 1e-3`; every numeric token
   agrees within `abs + rel·|ref|`.
 3. **a genuine wrong result is still caught** — perturbing one element by
   `+0.5` is REJECTED by the same tolerance. The gate is loose enough for
   last-ULP reordering, tight enough for a real bug.
 4. **the speedup is measured with the same machinery** — interleaved A/B,
   bootstrap 95% CI: **median 3.23×, CI [3.16, 3.26]**.
 The equivalence policy (`EquivalencePolicy::compare` in `diff-test`) is where
 the new rule lives: byte-identical stays exact; fp-tolerance tokenizes,
 compares numeric tokens within tolerance, handles NaN explicitly, and keeps
 non-numeric tokens and token counts exact. `fp-compare` applies a target's
 policy to two output files from the command line.
 **Verdict: `needs-human-review`** (`phase5-matmul-opt` ledger row). Using
 the FP-tolerance tier *at all* is a human-review signal per SPEC §8 — a
 person confirms the declared tolerance is defensible for the workload,
 exactly as with the mimalloc allocator swap. The machine measures and
 gates; the human ratifies the tolerance.
 ### Why this is the GPU lane in miniature
 The environment has no GPU, so the GPU fork is demonstrated on the CPU as
 what it actually is: **the same trust machinery with a different timer.** A
 Triton or CUDA kernel versus a reference kernel is the identical shape —
 reference-kernel differential testing within a tolerance, interleaved timing
 with a bootstrap CI, a ledger row routed to human review. Only the profiler
 and the bench clock become kernel-time-aware, and the hardware changes. The
 correctness story, the statistics, and the audit trail are already built and
 already exercised here on real code.
 Reproduce: `sh scripts/kernel-lane-demo.sh 512`.
 ---
 ## What Phase 5 settles
 The core loop was already complete at Phase 4. Phase 5 answers the "but does
 it generalize?" objection with the two hardest cases the spec named:
 - A **learned prior** can be extracted from the ledger without weakening a
  single gate — it only reorders *what to try first*.
 - A **numerically-reordered optimization** — the class byte-identical replay
  cannot handle — slots into the same harness by swapping the equivalence
  policy, with the correctness bar preserved (wrong results still caught) and
  the tolerance itself put under human review.
 Zero shipped false accepts, still. The product is trust, and trust
 transferred.
@@ -0,0 +1,32 @@
 # Kernel-lane demonstration — matmul (n=512, f32)
 ## 1. Equivalence: byte-identical is the wrong gate
 byte-identical: FAIL — 244901 of 262144 result values differ (accumulation reordered)
 diff-test FP-tolerance policy (targets/matmul/equivalence.toml):
  policy: FpTolerance { abs: 0.0001, rel: 0.001 }
  EQUIVALENT under policy
 in-process reference differential test:
  kernel differential test (opt vs reference):
    max_abs_err = 3.815e-05
    max_rel_err = 2.173e-02 (dominated by near-zero elements; see abs floor)
    tolerance   = abs 1.000e-04 + rel 1.000e-03 * |ref|
    byte_identical   = NO (accumulation reordered by design)
    within_tolerance = YES
    verdict = EQUIVALENT
 control — a genuine wrong result IS caught by the tolerance:
  tolerance correctly REJECTS a +0.5 perturbation of one element
 ## 2. Speedup: interleaved A/B, bootstrap 95% CI
  speedup (baseline/candidate): median 3.2301, 95% CI [3.1600, 3.2594]
  verdict: accepted
 ## What it demonstrates
 A real matmul speedup (transpose for cache locality + eight-accumulator
 ILP) that reorders the reduction. Byte-identical rejects it; the
 FP-tolerance gate accepts it (abs 1e-4 + rel 1e-3) while still catching
 a wrong result; the bench measures it with the same CI machinery as
 every other target. The GPU extension (Triton/CUDA vs a reference
 kernel) is the same shape — only the timer and the hardware change.
@@ -0,0 +1,17 @@
 learned class-selection ranking — evidence: 34 ledger rows, scope: all targets
 (Wilson 95% lower bound of the shippable-win rate; ties break untried-before-failed, then cheapest-first)
 rank  class         attempts  wins  held  win-rate  wilson-lb  note
 ------------------------------------------------------------------------------
   1  5 algorithmic       16     3     1       19%      0.066  proven winner
   2  3 alloc-churn        4     1     0       25%      0.046  proven winner
   3  1 build-config        9     1     0       11%      0.020  proven winner
   4  7 concurrency        0     0     0        0%      0.000  untried — cheapest-first prior only
   5  2 allocator          2     0     1        0%      0.000  held for human review (measured win, tier-gated)
   6  4 data-layout        1     0     0        0%      0.000  evidence against — tried, no shippable win
   7  6 simd               2     0     0        0%      0.000  evidence against — tried, no shippable win
 recommendation: try class 5 (algorithmic) first — best evidence (Wilson lb 0.066); then the next-ranked class whose profile preconditions match.
 Advisory only — the equivalence + significance gates decide every verdict. This ranking sharpens as the ledger grows.
 note: a 'win' counts only machine-sanitizer-verified accepts. Pre-sanitizer-gate accepts and the overturned comrak-010 are conservatively excluded — the policy trusts the ledger's machine record, not the external audit narrative.
@@ -0,0 +1,44 @@
 # Human ruling — `phase0-comrak-002` (mimalloc allocator swap)
 **Ruling: ACCEPTED, scoped to the measured workload.**
 **Ruled by:** project owner (repo maintainer), recorded 2026-07-12.
 **Ledger row:** `phase0-comrak-002` remains `needs-human-review` (the
 append-only ledger records what the *pipeline* decided; this document is
 the human decision layered on top).
 ## What was measured
 Swapping comrak's global allocator to
 [mimalloc](https://github.com/microsoft/mimalloc):
 - **Speedup: 1.046 median, 95% CI [1.033, 1.058]** (+4.6%) on the comrak
  CLI rendering the pinned Pro Git corpus (11.0 MB markdown, single
  stream), interleaved A/B vs a pristine-rebuilt baseline.
 - Byte-identical golden replay. The pipeline routed it to
  `needs-human-review` per SPEC §8 because a global allocator swap is a
  behavioral change no single benchmark can fully clear.
 ## The ruling and its scope
 **Accepted** as a real, CI-significant win **for the comrak CLI's
 single-stream markdown-render workload** — the workload measured, on the
 calibrated hardware in `results/calibration/`.
 This is explicitly **not** a blanket "always link mimalloc" mandate:
 - The upside (+4.6%) is genuine but modest; it does not justify making
  mimalloc a default across unrelated targets or workloads without their
  own measurement.
 - The comrak CLI on this workload is single-threaded, which is what makes
  the ruling low-risk here — the concurrency/fragmentation concerns SPEC
  §8 flags are minimal for a short-lived, single-stream CLI process. A
  long-running or heavily-multithreaded comrak *library* embedding would
  need its own review before inheriting this.
 ## Why this is the honest resolution
 The `needs-human-review` tier exists precisely to force this judgment
 instead of letting a green benchmark auto-ship a global behavioral
 change. The judgment: yes for the measured CLI workload, with the scope
 written down so the number can't be over-generalized later. The tier did
 its job; the win is real; the ruling is bounded.
@@ -0,0 +1,60 @@
 #!/bin/sh
 # Kernel-lane demonstration (SPEC §13 research fork). Shows the one thing
 # the parser/service targets never needed: an optimization that reorders
 # floating-point accumulation, so byte-identical golden replay is the WRONG
 # gate and a floating-point TOLERANCE gate is required — while a genuine
 # wrong result is still caught. Measured with the same interleaved A/B +
 # bootstrap CI as every other target.
 #
 #   sh scripts/kernel-lane-demo.sh [n]
 set -eu
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
 N=${1:-512}
 OUT="$ROOT/results/phase5"
 mkdir -p "$OUT" /tmp/kl
 cd "$ROOT"
 clang -O2 -o /tmp/kl/kernel targets/matmul/kernel.c -lm
 cargo build -q -p diff-test -p bench-runner
 {
 echo "# Kernel-lane demonstration — matmul (n=$N, f32)"
 echo
 echo "## 1. Equivalence: byte-identical is the wrong gate"
 /tmp/kl/kernel "$N" emit ref > /tmp/kl/ref.txt
 /tmp/kl/kernel "$N" emit opt > /tmp/kl/opt.txt
 if diff -q /tmp/kl/ref.txt /tmp/kl/opt.txt >/dev/null; then
    echo "byte-identical: MATCH (unexpected — no reorder)"
 else
    d=$(diff /tmp/kl/ref.txt /tmp/kl/opt.txt | grep -c '^<' || true)
    echo "byte-identical: FAIL — $d of $((N*N)) result values differ (accumulation reordered)"
 fi
 echo
 echo "diff-test FP-tolerance policy (targets/matmul/equivalence.toml):"
 "$ROOT/target/debug/fp-compare" targets/matmul /tmp/kl/ref.txt /tmp/kl/opt.txt | sed 's/^/  /'
 echo
 echo "in-process reference differential test:"
 /tmp/kl/kernel "$N" check 1e-4 1e-3 | sed 's/^/  /'
 echo
 echo "control — a genuine wrong result IS caught by the tolerance:"
 awk 'NR==1{$1=$1+0.5} {print}' /tmp/kl/opt.txt > /tmp/kl/wrong.txt
 if "$ROOT/target/debug/fp-compare" targets/matmul /tmp/kl/ref.txt /tmp/kl/wrong.txt >/dev/null 2>&1; then
    echo "  !! tolerance accepted a wrong result (bug)"
 else
    echo "  tolerance correctly REJECTS a +0.5 perturbation of one element"
 fi
 echo
 echo "## 2. Speedup: interleaved A/B, bootstrap 95% CI"
 cargo run -q -p bench-runner -- --config config/accept.toml compare \
    --baseline "taskset -c 2 /tmp/kl/kernel $N run ref" \
    --candidate "taskset -c 2 /tmp/kl/kernel $N run opt" 2>/dev/null \
    | grep -E "speedup|verdict" | sed 's/^/  /'
 echo
 echo "## What it demonstrates"
 echo "A real matmul speedup (transpose for cache locality + eight-accumulator"
 echo "ILP) that reorders the reduction. Byte-identical rejects it; the"
 echo "FP-tolerance gate accepts it (abs 1e-4 + rel 1e-3) while still catching"
 echo "a wrong result; the bench measures it with the same CI machinery as"
 echo "every other target. The GPU extension (Triton/CUDA vs a reference"
 echo "kernel) is the same shape — only the timer and the hardware change."
 } | tee "$OUT/kernel-lane.txt"
@@ -0,0 +1,14 @@
 # Kernel-lane equivalence policy (SPEC §13, §3.2). The blocked matmul
 # reorders floating-point accumulation, so byte-identical golden replay is
 # the WRONG gate — it would reject a correct optimization for differing in
 # the last ULPs. Instead, numeric outputs must agree within tolerance;
 # anything non-numeric still matches exactly, and a genuine wrong result
 # (beyond the tolerance) is still caught.
 #
 # rel = 1e-3 is generous for f32 accumulation over 512 terms in a different
 # order (observed max relative error ~1e-4); a real bug moves it far past
 # this. Using this tier at all is itself a needs-human-review signal per
 # SPEC §8 — a human confirms the tolerance is defensible for the workload.
 mode = "fp-tolerance"
 abs = 1e-4
 rel = 1e-3
@@ -0,0 +1,180 @@
 /* hotpath kernel-lane demonstration (SPEC §13).
 *
 * A single-precision matrix multiply in two implementations that are
 * mathematically identical but numerically different because they sum the
 * inner products in a different ORDER:
 *
 *   matmul_ref — naive i,j,k with a single sequential accumulator. The
 *                reference oracle.
 *   matmul_opt — eight independent accumulators over the k-reduction,
 *                tree-combined at the end. This breaks the float-add
 *                dependency chain (hiding ~4-cycle add latency behind
 *                instruction-level parallelism, so it is faster) AND sums
 *                the inner product in a different ORDER, so results differ
 *                in the last few ULPs — bit-identical is impossible by
 *                construction.
 *
 * This is the crux of the kernel lane: a real optimization here CANNOT be
 * gated by byte-identical golden replay (the low bits differ by design);
 * it needs a floating-point *tolerance* gate that accepts last-ULP
 * reordering while still catching a genuine wrong result. The reference
 * implementation is the oracle; the candidate is differential-tested
 * against it within a declared tolerance. On a GPU this same shape holds
 * (a Triton/CUDA kernel vs a reference kernel) — only the timer and the
 * hardware change.
 *
 *   kernel <n> emit  ref|opt              # print the result matrix
 *   kernel <n> check <abs-tol> <rel-tol>  # in-process differential test
 *   kernel <n> run   ref|opt              # one matmul, no output (for A/B)
 *   kernel <n> bench ref|opt <reps>       # time one implementation
 */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <math.h>
 #include <time.h>
 #define BS 64 /* block size */
 /* Deterministic f32 fill — xorshift, identical on every machine. */
 static void fill(float *m, long n, unsigned seed) {
    unsigned s = seed ? seed : 1;
    for (long i = 0; i < n * n; i++) {
        s ^= s << 13; s ^= s >> 17; s ^= s << 5;
        /* in [-1, 1) */
        m[i] = (float)((double)s / 2147483648.0 - 1.0);
    }
 }
 static void matmul_ref(const float *A, const float *B, float *C, long n) {
    for (long i = 0; i < n; i++)
        for (long j = 0; j < n; j++) {
            float acc = 0.0f;
            for (long k = 0; k < n; k++)
                acc += A[i * n + k] * B[k * n + j];
            C[i * n + j] = acc;
        }
 }
 static void matmul_opt(const float *A, const float *B, float *C, long n) {
    /* Transpose B so the inner reduction walks BT[j] contiguously instead
     * of striding B by n (the cache win), and use eight independent
     * accumulators (the ILP win, which also REORDERS the reduction). Both
     * levers together are a robust speedup AND make bit-identical output
     * impossible — the kernel lane in one function. */
    float *BT = malloc((size_t)n * n * sizeof(float));
    if (!BT) {
        matmul_ref(A, B, C, n);
        return;
    }
    for (long i = 0; i < n; i++)
        for (long j = 0; j < n; j++)
            BT[j * n + i] = B[i * n + j];
    for (long i = 0; i < n; i++) {
        const float *a = &A[i * n];
        for (long j = 0; j < n; j++) {
            const float *b = &BT[j * n];
            float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0};
            long k = 0;
            for (; k + 8 <= n; k += 8)
                for (int r = 0; r < 8; r++)
                    acc[r] += a[k + r] * b[k + r];
            float s = ((acc[0] + acc[1]) + (acc[2] + acc[3]))
                    + ((acc[4] + acc[5]) + (acc[6] + acc[7]));
            for (; k < n; k++)
                s += a[k] * b[k];
            C[i * n + j] = s;
        }
    }
    free(BT);
 }
 static double now_s(void) {
    struct timespec t;
    clock_gettime(CLOCK_MONOTONIC, &t);
    return (double)t.tv_sec + (double)t.tv_nsec / 1e9;
 }
 int main(int argc, char **argv) {
    if (argc < 3) {
        fprintf(stderr, "usage: %s <n> emit ref|opt | check <abs-tol> <rel-tol> | run ref|opt | bench ref|opt <reps>\n", argv[0]);
        return 2;
    }
    long n = atol(argv[1]);
    if (n < 1) n = 512;
    const char *mode = argv[2];
    float *A = malloc((size_t)n * n * sizeof(float));
    float *B = malloc((size_t)n * n * sizeof(float));
    float *C = malloc((size_t)n * n * sizeof(float));
    float *R = malloc((size_t)n * n * sizeof(float));
    if (!A || !B || !C || !R) return 2;
    fill(A, n, 0x9E3779B1u);
    fill(B, n, 0x85EBCA77u);
    if (strcmp(mode, "emit") == 0) {
        int blocked = (argc > 3 && strcmp(argv[3], "opt") == 0);
        if (blocked) matmul_opt(A, B, C, n);
        else matmul_ref(A, B, C, n);
        for (long i = 0; i < n * n; i++) printf("%.9g\n", C[i]);
        return 0;
    }
    if (strcmp(mode, "check") == 0) {
        /* Combined abs+rel tolerance — the standard for FP comparison, and
         * what the diff-test policy uses. Rel-only is wrong here: result
         * elements near zero (catastrophic cancellation) have meaningless
         * relative error, so an abs floor is required. */
        double abs_tol = argc > 3 ? atof(argv[3]) : 1e-4;
        double rel_tol = argc > 4 ? atof(argv[4]) : 1e-3;
        matmul_ref(A, B, R, n);
        matmul_opt(A, B, C, n);
        double max_abs = 0.0, max_rel = 0.0, worst_excess = -1e30;
        for (long i = 0; i < n * n; i++) {
            double d = fabs((double)C[i] - (double)R[i]);
            double r = d / (fabs((double)R[i]) + 1e-30);
            double tol = abs_tol + rel_tol * fabs((double)R[i]);
            if (d > max_abs) max_abs = d;
            if (r > max_rel) max_rel = r;
            if (d - tol > worst_excess) worst_excess = d - tol;
        }
        int pass = worst_excess <= 0.0;
        printf("kernel differential test (opt vs reference):\n");
        printf("  max_abs_err = %.3e\n", max_abs);
        printf("  max_rel_err = %.3e (dominated by near-zero elements; see abs floor)\n", max_rel);
        printf("  tolerance   = abs %.3e + rel %.3e * |ref|\n", abs_tol, rel_tol);
        printf("  byte_identical   = NO (accumulation reordered by design)\n");
        printf("  within_tolerance = %s\n", pass ? "YES" : "NO");
        printf("  verdict = %s\n", pass ? "EQUIVALENT" : "DIVERGENT");
        return pass ? 0 : 1;
    }
    /* One matmul, no output — for bench-runner's interleaved A/B (external
     * wall-clock timing with a bootstrap CI, like every other target). */
    if (strcmp(mode, "run") == 0) {
        if (strcmp(argv[3], "opt") == 0) matmul_opt(A, B, C, n);
        else matmul_ref(A, B, C, n);
        /* touch C so the compiler cannot elide the work */
        volatile float sink = C[n * n - 1];
        (void)sink;
        return 0;
    }
    if (strcmp(mode, "bench") == 0) {
        int use_opt = strcmp(argv[3], "opt") == 0;
        long reps = argc > 4 ? atol(argv[4]) : 5;
        if (use_opt) matmul_opt(A, B, C, n); else matmul_ref(A, B, C, n);
        double best = 1e30;
        for (long r = 0; r < reps; r++) {
            double t0 = now_s();
            if (use_opt) matmul_opt(A, B, C, n); else matmul_ref(A, B, C, n);
            double dt = now_s() - t0;
            if (dt < best) best = dt;
        }
        printf("%.6f\n", best);
        return 0;
    }
    fprintf(stderr, "unknown mode: %s\n", mode);
    return 2;
 }
		`@@ -0,0 +1 @@`
							`{{#include ../../../results/phase3/02-build-flags-and-allocators.md}}`
		`@@ -0,0 +1 @@`
							`{{#include ../../../results/phase3/04-cjson-c-target.md}}`
		`@@ -0,0 +1 @@`
							`{{#include ../../../results/phase3/03-comrak-caught-false-accept.md}}`
		`@@ -0,0 +1 @@`
							`{{#include ../../../results/phase4/README.md}}`
		`@@ -0,0 +1 @@`
							`{{#include ../../../results/phase3/01-tokei-algorithmic-rust.md}}`