phase5: learned class-selection policy (SPEC §13) — rank classes by ledger evidence

crates/policy: reads the append-only ledger and ranks playbook classes by
the Wilson score lower bound of their machine-verified win rate — the same
'quote the lower bound' discipline the rest of the project uses. A class
that won 1/1 does not outrank one that won 8/10 (wider interval); untried
classes fall back to the cheapest-first prior; tried-and-failed classes
rank last. Advisory only — the gates still decide every verdict.

On the current 33-row ledger it ranks class 5 (algorithmic) first, then
class 3, then class 1 — the proven winners — with class 7 (untried) above
the evidence-against classes. Wins count only machine-sanitizer-verified
accepts (conservatively excluding pre-gate accepts and the overturned
comrak-010): the policy trusts the ledger's machine record, not narrative.
4 unit tests (Wilson monotonicity, small-sample penalty, ranking order).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2

Cargo.lock

@@ -351,6 +351,15 @@ version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
+[[package]]
+name = "policy"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "ledger",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"

Cargo.toml

@@ -9,6 +9,7 @@ members = [
     "crates/report",
     "crates/verdict",
     "crates/harnessd",
+    "crates/policy",
 ]
 
 [workspace.package]

crates/ledger/src/lib.rs

@@ -190,6 +190,28 @@ impl Ledger {
     /// onto a completed attempt (harnessd `read_verdict`). Returns the
     /// verdict plus the bench CIs; never enough to game, since the row
     /// is already written and immutable.
+    /// Every attempt's (playbook_class, target, verdict, sanitizers_clean)
+    /// — the training signal for the learned class-selection policy
+    /// (SPEC §13). A "shippable win" is `accepted` AND sanitizers-clean,
+    /// matching the report generator's definition.
+    pub fn all_outcomes(&self) -> Result<Vec<(u8, String, String, bool)>, LedgerError> {
+        let mut stmt = self
+            .conn
+            .prepare("SELECT playbook_class, target, verdict, gates FROM attempts")?;
+        let rows = stmt.query_map([], |r| {
+            let class: u8 = r.get(0)?;
+            let target: String = r.get(1)?;
+            let verdict: String = r.get(2)?;
+            let gates: Option<String> = r.get(3)?;
+            let sanitizers_clean = gates
+                .and_then(|g| serde_json::from_str::<serde_json::Value>(&g).ok())
+                .and_then(|g| g.get("sanitizers_clean").and_then(|v| v.as_bool()))
+                .unwrap_or(false);
+            Ok((class, target, verdict, sanitizers_clean))
+        })?;
+        Ok(rows.collect::<Result<Vec<_>, _>>()?)
+    }
+
     /// Full row data the ROI report generator needs (SPEC §9). Includes
     /// the workload string pulled out of the bench evidence.
     pub fn report_row(&self, run_id: &str) -> Result<Option<serde_json::Value>, LedgerError> {

crates/policy/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "policy"
+description = "Learned class-selection policy (SPEC §13): rank playbook classes by the Wilson lower bound of their observed win-rate in the ledger, degrading to cheapest-first when data is thin"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+clap.workspace = true
+ledger = { path = "../ledger" }

crates/policy/src/lib.rs

@@ -0,0 +1,162 @@
+//! Learned class-selection policy (SPEC §13 research fork).
+//!
+//! The append-only ledger is a growing dataset of `(playbook_class,
+//! target) -> verdict`. This turns that history into a *prior*: given a
+//! target, rank the playbook classes by how likely each is to yield a
+//! shippable win — learned from evidence, not the fixed cheapest-first
+//! order alone.
+//!
+//! The ranking statistic is the **Wilson score lower bound** of the
+//! win-rate, not the point estimate. This is the same discipline the rest
+//! of the project uses (quote the CI lower bound, never the median): a
+//! class that won 1/1 does NOT outrank one that won 8/10, because its
+//! interval is wide. With no data a class scores 0 and falls back to the
+//! cheapest-first prior. As the ledger grows, the estimates sharpen — the
+//! policy is honest about its own uncertainty by construction.
+
+/// Wilson score interval lower bound for `wins`/`n` at z (1.96 = 95%).
+/// Returns 0.0 for n == 0 (no evidence).
+pub fn wilson_lower_bound(wins: u64, n: u64, z: f64) -> f64 {
+    if n == 0 {
+        return 0.0;
+    }
+    let n = n as f64;
+    let p = wins as f64 / n;
+    let z2 = z * z;
+    let denom = 1.0 + z2 / n;
+    let center = p + z2 / (2.0 * n);
+    let margin = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt();
+    ((center - margin) / denom).max(0.0)
+}
+
+/// Aggregated evidence for one playbook class.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ClassStat {
+    pub class: u8,
+    pub attempts: u64,
+    /// Shippable wins: `accepted` AND sanitizers-clean.
+    pub wins: u64,
+    /// Real wins held for a human ruling (needs-human-review) — counted
+    /// separately, since they are neither clean auto-wins nor failures.
+    pub held: u64,
+    pub win_rate: f64,
+    pub wilson_lb: f64,
+}
+
+impl ClassStat {
+    /// A short human note on the evidence state.
+    pub fn note(&self) -> &'static str {
+        if self.attempts == 0 {
+            "untried — cheapest-first prior only"
+        } else if self.wins > 0 {
+            "proven winner"
+        } else if self.held > 0 {
+            "held for human review (measured win, tier-gated)"
+        } else {
+            "evidence against — tried, no shippable win"
+        }
+    }
+}
+
+/// Build per-class stats (classes 1..=7) from ledger outcome rows,
+/// optionally filtered to one target. Each outcome is
+/// `(class, target, verdict, sanitizers_clean)`.
+pub fn class_stats(
+    outcomes: &[(u8, String, String, bool)],
+    target: Option<&str>,
+    z: f64,
+) -> Vec<ClassStat> {
+    (1u8..=7)
+        .map(|class| {
+            let rows = outcomes
+                .iter()
+                .filter(|(c, t, _, _)| *c == class && target.is_none_or(|want| t == want));
+            let (mut attempts, mut wins, mut held) = (0u64, 0u64, 0u64);
+            for (_, _, verdict, san) in rows {
+                attempts += 1;
+                if verdict == "accepted" && *san {
+                    wins += 1;
+                } else if verdict == "needs-human-review" {
+                    held += 1;
+                }
+            }
+            let win_rate = if attempts > 0 {
+                wins as f64 / attempts as f64
+            } else {
+                0.0
+            };
+            ClassStat {
+                class,
+                attempts,
+                wins,
+                held,
+                win_rate,
+                wilson_lb: wilson_lower_bound(wins, attempts, z),
+            }
+        })
+        .collect()
+}
+
+/// Rank classes for selection. Primary key: Wilson lower bound (evidence
+/// of success), descending. Ties (notably all the zero-evidence classes)
+/// break by: fewer failed attempts first (prefer untried over
+/// proven-failure), then cheaper class first (the cost prior).
+pub fn rank(mut stats: Vec<ClassStat>) -> Vec<ClassStat> {
+    stats.sort_by(|a, b| {
+        b.wilson_lb
+            .partial_cmp(&a.wilson_lb)
+            .unwrap_or(std::cmp::Ordering::Equal)
+            .then_with(|| {
+                // failures = attempts without a win or a hold
+                let fa = a.attempts - a.wins - a.held;
+                let fb = b.attempts - b.wins - b.held;
+                fa.cmp(&fb)
+            })
+            .then_with(|| a.class.cmp(&b.class))
+    });
+    stats
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn wilson_is_zero_without_evidence() {
+        assert_eq!(wilson_lower_bound(0, 0, 1.96), 0.0);
+    }
+
+    #[test]
+    fn wilson_penalizes_small_samples() {
+        // 1/1 must NOT outrank 8/10 — the small sample's interval is wide.
+        let one_of_one = wilson_lower_bound(1, 1, 1.96);
+        let eight_of_ten = wilson_lower_bound(8, 10, 1.96);
+        assert!(
+            one_of_one < eight_of_ten,
+            "1/1 ({one_of_one:.3}) should rank below 8/10 ({eight_of_ten:.3})"
+        );
+    }
+
+    #[test]
+    fn wilson_monotonic_in_evidence() {
+        // Same rate, more evidence → higher lower bound.
+        assert!(wilson_lower_bound(50, 100, 1.96) > wilson_lower_bound(5, 10, 1.96));
+    }
+
+    #[test]
+    fn rank_prefers_proven_then_untried_then_failed() {
+        let outcomes = vec![
+            (5u8, "t".into(), "accepted".into(), true),
+            (5u8, "t".into(), "accepted".into(), true),
+            (5u8, "t".into(), "rejected-bench".into(), false),
+            (6u8, "t".into(), "rejected-bench".into(), false), // evidence against
+                                                               // class 7 untried
+        ];
+        let ranked = rank(class_stats(&outcomes, None, 1.96));
+        assert_eq!(ranked[0].class, 5, "proven class 5 ranks first");
+        // untried class (7) must rank above the tried-and-failed class 6
+        let pos7 = ranked.iter().position(|s| s.class == 7).unwrap();
+        let pos6 = ranked.iter().position(|s| s.class == 6).unwrap();
+        assert!(pos7 < pos6, "untried class 7 outranks failed class 6");
+    }
+}

crates/policy/src/main.rs

@@ -0,0 +1,104 @@
+//! `policy rank` — the learned class-selection prior (SPEC §13).
+//! Reads the append-only ledger and ranks playbook classes by the Wilson
+//! lower bound of their observed shippable-win rate, degrading to
+//! cheapest-first where evidence is thin. Advisory only: the gates still
+//! decide every verdict. This just tells the agent where to look first.
+
+use anyhow::Result;
+use clap::Parser;
+use policy::{class_stats, rank};
+use std::path::PathBuf;
+
+const CLASS_NAME: [&str; 8] = [
+    "",
+    "build-config",
+    "allocator",
+    "alloc-churn",
+    "data-layout",
+    "algorithmic",
+    "simd",
+    "concurrency",
+];
+
+#[derive(Parser)]
+#[command(
+    name = "policy",
+    about = "Learned playbook-class ranking from the ledger (SPEC §13)"
+)]
+struct Cli {
+    /// Restrict the evidence to one target (else pool all targets).
+    #[arg(long)]
+    target: Option<String>,
+    #[arg(long, default_value = "results/ledger.sqlite")]
+    db: PathBuf,
+    /// z for the Wilson interval (1.96 = 95%).
+    #[arg(long, default_value_t = 1.96)]
+    z: f64,
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let ledger = ledger::Ledger::open(&cli.db)?;
+    let outcomes = ledger.all_outcomes()?;
+    let ranked = rank(class_stats(&outcomes, cli.target.as_deref(), cli.z));
+
+    let scope = cli.target.as_deref().unwrap_or("all targets");
+    println!(
+        "learned class-selection ranking — evidence: {} ledger rows, scope: {scope}\n\
+         (Wilson 95% lower bound of the shippable-win rate; ties break untried-before-failed, then cheapest-first)\n",
+        outcomes.len()
+    );
+    println!(
+        "{:>4}  {:<13} {:>8} {:>5} {:>5} {:>9} {:>10}  note",
+        "rank", "class", "attempts", "wins", "held", "win-rate", "wilson-lb"
+    );
+    println!("{}", "-".repeat(78));
+    for (i, s) in ranked.iter().enumerate() {
+        println!(
+            "{:>4}  {:>1} {:<11} {:>8} {:>5} {:>5} {:>8.0}% {:>10.3}  {}",
+            i + 1,
+            s.class,
+            CLASS_NAME[s.class as usize],
+            s.attempts,
+            s.wins,
+            s.held,
+            s.win_rate * 100.0,
+            s.wilson_lb,
+            s.note(),
+        );
+    }
+
+    // The actionable line: the first class with real supporting evidence,
+    // else the cheapest untried class.
+    let best_evidenced = ranked.iter().find(|s| s.wilson_lb > 0.0);
+    let cheapest_untried = ranked
+        .iter()
+        .filter(|s| s.attempts == 0)
+        .min_by_key(|s| s.class);
+    println!();
+    match (best_evidenced, cheapest_untried) {
+        (Some(e), _) => println!(
+            "recommendation: try class {} ({}) first — best evidence (Wilson lb {:.3}); \
+             then the next-ranked class whose profile preconditions match.",
+            e.class, CLASS_NAME[e.class as usize], e.wilson_lb
+        ),
+        (None, Some(u)) => println!(
+            "recommendation: no class has supporting evidence yet; fall back to the \
+             cheapest untried class {} ({}).",
+            u.class, CLASS_NAME[u.class as usize]
+        ),
+        (None, None) => {
+            println!("recommendation: no evidence and no untried class; use cheapest-first.")
+        }
+    }
+    println!(
+        "\nAdvisory only — the equivalence + significance gates decide every verdict. \
+         This ranking sharpens as the ledger grows."
+    );
+    println!(
+        "note: a 'win' counts only machine-sanitizer-verified accepts. Pre-sanitizer-gate \
+         accepts and the overturned comrak-010 are conservatively excluded — the policy \
+         trusts the ledger's machine record, not the external audit narrative."
+    );
+    Ok(())
+}

justfile

@@ -84,3 +84,7 @@ service-calibrate server doc out *flags:
 # dollars and/or latency percentiles, CIs + methodology printed inline.
 report run_id *flags:
     cargo run -p report -- --run-id "{{run_id}}" {{flags}}
+
+# Learned playbook-class ranking from the ledger (SPEC §13 research fork)
+policy *flags:
+    cargo run -p policy -- {{flags}}