phase5: learned class-selection policy (SPEC §13) — rank classes by ledger evidence

crates/policy: reads the append-only ledger and ranks playbook classes by
the Wilson score lower bound of their machine-verified win rate — the same
'quote the lower bound' discipline the rest of the project uses. A class
that won 1/1 does not outrank one that won 8/10 (wider interval); untried
classes fall back to the cheapest-first prior; tried-and-failed classes
rank last. Advisory only — the gates still decide every verdict.

On the current 33-row ledger it ranks class 5 (algorithmic) first, then
class 3, then class 1 — the proven winners — with class 7 (untried) above
the evidence-against classes. Wins count only machine-sanitizer-verified
accepts (conservatively excluding pre-gate accepts and the overturned
comrak-010): the policy trusts the ledger's machine record, not narrative.
4 unit tests (Wilson monotonicity, small-sample penalty, ranking order).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2

crates/policy/src/lib.rs

@@ -0,0 +1,162 @@
+//! Learned class-selection policy (SPEC §13 research fork).
+//!
+//! The append-only ledger is a growing dataset of `(playbook_class,
+//! target) -> verdict`. This turns that history into a *prior*: given a
+//! target, rank the playbook classes by how likely each is to yield a
+//! shippable win — learned from evidence, not the fixed cheapest-first
+//! order alone.
+//!
+//! The ranking statistic is the **Wilson score lower bound** of the
+//! win-rate, not the point estimate. This is the same discipline the rest
+//! of the project uses (quote the CI lower bound, never the median): a
+//! class that won 1/1 does NOT outrank one that won 8/10, because its
+//! interval is wide. With no data a class scores 0 and falls back to the
+//! cheapest-first prior. As the ledger grows, the estimates sharpen — the
+//! policy is honest about its own uncertainty by construction.
+
+/// Wilson score interval lower bound for `wins`/`n` at z (1.96 = 95%).
+/// Returns 0.0 for n == 0 (no evidence).
+pub fn wilson_lower_bound(wins: u64, n: u64, z: f64) -> f64 {
+    if n == 0 {
+        return 0.0;
+    }
+    let n = n as f64;
+    let p = wins as f64 / n;
+    let z2 = z * z;
+    let denom = 1.0 + z2 / n;
+    let center = p + z2 / (2.0 * n);
+    let margin = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt();
+    ((center - margin) / denom).max(0.0)
+}
+
+/// Aggregated evidence for one playbook class.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ClassStat {
+    pub class: u8,
+    pub attempts: u64,
+    /// Shippable wins: `accepted` AND sanitizers-clean.
+    pub wins: u64,
+    /// Real wins held for a human ruling (needs-human-review) — counted
+    /// separately, since they are neither clean auto-wins nor failures.
+    pub held: u64,
+    pub win_rate: f64,
+    pub wilson_lb: f64,
+}
+
+impl ClassStat {
+    /// A short human note on the evidence state.
+    pub fn note(&self) -> &'static str {
+        if self.attempts == 0 {
+            "untried — cheapest-first prior only"
+        } else if self.wins > 0 {
+            "proven winner"
+        } else if self.held > 0 {
+            "held for human review (measured win, tier-gated)"
+        } else {
+            "evidence against — tried, no shippable win"
+        }
+    }
+}
+
+/// Build per-class stats (classes 1..=7) from ledger outcome rows,
+/// optionally filtered to one target. Each outcome is
+/// `(class, target, verdict, sanitizers_clean)`.
+pub fn class_stats(
+    outcomes: &[(u8, String, String, bool)],
+    target: Option<&str>,
+    z: f64,
+) -> Vec<ClassStat> {
+    (1u8..=7)
+        .map(|class| {
+            let rows = outcomes
+                .iter()
+                .filter(|(c, t, _, _)| *c == class && target.is_none_or(|want| t == want));
+            let (mut attempts, mut wins, mut held) = (0u64, 0u64, 0u64);
+            for (_, _, verdict, san) in rows {
+                attempts += 1;
+                if verdict == "accepted" && *san {
+                    wins += 1;
+                } else if verdict == "needs-human-review" {
+                    held += 1;
+                }
+            }
+            let win_rate = if attempts > 0 {
+                wins as f64 / attempts as f64
+            } else {
+                0.0
+            };
+            ClassStat {
+                class,
+                attempts,
+                wins,
+                held,
+                win_rate,
+                wilson_lb: wilson_lower_bound(wins, attempts, z),
+            }
+        })
+        .collect()
+}
+
+/// Rank classes for selection. Primary key: Wilson lower bound (evidence
+/// of success), descending. Ties (notably all the zero-evidence classes)
+/// break by: fewer failed attempts first (prefer untried over
+/// proven-failure), then cheaper class first (the cost prior).
+pub fn rank(mut stats: Vec<ClassStat>) -> Vec<ClassStat> {
+    stats.sort_by(|a, b| {
+        b.wilson_lb
+            .partial_cmp(&a.wilson_lb)
+            .unwrap_or(std::cmp::Ordering::Equal)
+            .then_with(|| {
+                // failures = attempts without a win or a hold
+                let fa = a.attempts - a.wins - a.held;
+                let fb = b.attempts - b.wins - b.held;
+                fa.cmp(&fb)
+            })
+            .then_with(|| a.class.cmp(&b.class))
+    });
+    stats
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn wilson_is_zero_without_evidence() {
+        assert_eq!(wilson_lower_bound(0, 0, 1.96), 0.0);
+    }
+
+    #[test]
+    fn wilson_penalizes_small_samples() {
+        // 1/1 must NOT outrank 8/10 — the small sample's interval is wide.
+        let one_of_one = wilson_lower_bound(1, 1, 1.96);
+        let eight_of_ten = wilson_lower_bound(8, 10, 1.96);
+        assert!(
+            one_of_one < eight_of_ten,
+            "1/1 ({one_of_one:.3}) should rank below 8/10 ({eight_of_ten:.3})"
+        );
+    }
+
+    #[test]
+    fn wilson_monotonic_in_evidence() {
+        // Same rate, more evidence → higher lower bound.
+        assert!(wilson_lower_bound(50, 100, 1.96) > wilson_lower_bound(5, 10, 1.96));
+    }
+
+    #[test]
+    fn rank_prefers_proven_then_untried_then_failed() {
+        let outcomes = vec![
+            (5u8, "t".into(), "accepted".into(), true),
+            (5u8, "t".into(), "accepted".into(), true),
+            (5u8, "t".into(), "rejected-bench".into(), false),
+            (6u8, "t".into(), "rejected-bench".into(), false), // evidence against
+                                                               // class 7 untried
+        ];
+        let ranked = rank(class_stats(&outcomes, None, 1.96));
+        assert_eq!(ranked[0].class, 5, "proven class 5 ranks first");
+        // untried class (7) must rank above the tried-and-failed class 6
+        let pos7 = ranked.iter().position(|s| s.class == 7).unwrap();
+        let pos6 = ranked.iter().position(|s| s.class == 6).unwrap();
+        assert!(pos7 < pos6, "untried class 7 outranks failed class 6");
+    }
+}