From b7297ad31f4a9e9b34cb21bf2b4c45719694a25d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 00:24:00 +0000 Subject: [PATCH] =?UTF-8?q?phase5:=20learned=20class-selection=20policy=20?= =?UTF-8?q?(SPEC=20=C2=A713)=20=E2=80=94=20rank=20classes=20by=20ledger=20?= =?UTF-8?q?evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit crates/policy: reads the append-only ledger and ranks playbook classes by the Wilson score lower bound of their machine-verified win rate — the same 'quote the lower bound' discipline the rest of the project uses. A class that won 1/1 does not outrank one that won 8/10 (wider interval); untried classes fall back to the cheapest-first prior; tried-and-failed classes rank last. Advisory only — the gates still decide every verdict. On the current 33-row ledger it ranks class 5 (algorithmic) first, then class 3, then class 1 — the proven winners — with class 7 (untried) above the evidence-against classes. Wins count only machine-sanitizer-verified accepts (conservatively excluding pre-gate accepts and the overturned comrak-010): the policy trusts the ledger's machine record, not narrative. 4 unit tests (Wilson monotonicity, small-sample penalty, ranking order). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2 --- Cargo.lock | 9 +++ Cargo.toml | 1 + crates/ledger/src/lib.rs | 22 ++++++ crates/policy/Cargo.toml | 12 +++ crates/policy/src/lib.rs | 162 ++++++++++++++++++++++++++++++++++++++ crates/policy/src/main.rs | 104 ++++++++++++++++++++++++ justfile | 4 + 7 files changed, 314 insertions(+) create mode 100644 crates/policy/Cargo.toml create mode 100644 crates/policy/src/lib.rs create mode 100644 crates/policy/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 0658d93..069b70d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,6 +351,15 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "policy" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "ledger", +] + [[package]] name = "proc-macro2" version = "1.0.106" diff --git a/Cargo.toml b/Cargo.toml index ed3d792..bccb3b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/report", "crates/verdict", "crates/harnessd", + "crates/policy", ] [workspace.package] diff --git a/crates/ledger/src/lib.rs b/crates/ledger/src/lib.rs index 4b8fd8c..71459b6 100644 --- a/crates/ledger/src/lib.rs +++ b/crates/ledger/src/lib.rs @@ -190,6 +190,28 @@ impl Ledger { /// onto a completed attempt (harnessd `read_verdict`). Returns the /// verdict plus the bench CIs; never enough to game, since the row /// is already written and immutable. + /// Every attempt's (playbook_class, target, verdict, sanitizers_clean) + /// — the training signal for the learned class-selection policy + /// (SPEC §13). A "shippable win" is `accepted` AND sanitizers-clean, + /// matching the report generator's definition. + pub fn all_outcomes(&self) -> Result, LedgerError> { + let mut stmt = self + .conn + .prepare("SELECT playbook_class, target, verdict, gates FROM attempts")?; + let rows = stmt.query_map([], |r| { + let class: u8 = r.get(0)?; + let target: String = r.get(1)?; + let verdict: String = r.get(2)?; + let gates: Option = r.get(3)?; + let sanitizers_clean = gates + .and_then(|g| serde_json::from_str::(&g).ok()) + .and_then(|g| g.get("sanitizers_clean").and_then(|v| v.as_bool())) + .unwrap_or(false); + Ok((class, target, verdict, sanitizers_clean)) + })?; + Ok(rows.collect::, _>>()?) + } + /// Full row data the ROI report generator needs (SPEC §9). Includes /// the workload string pulled out of the bench evidence. pub fn report_row(&self, run_id: &str) -> Result, LedgerError> { diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml new file mode 100644 index 0000000..9c2320b --- /dev/null +++ b/crates/policy/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "policy" +description = "Learned class-selection policy (SPEC §13): rank playbook classes by the Wilson lower bound of their observed win-rate in the ledger, degrading to cheapest-first when data is thin" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ledger = { path = "../ledger" } diff --git a/crates/policy/src/lib.rs b/crates/policy/src/lib.rs new file mode 100644 index 0000000..2a18264 --- /dev/null +++ b/crates/policy/src/lib.rs @@ -0,0 +1,162 @@ +//! Learned class-selection policy (SPEC §13 research fork). +//! +//! The append-only ledger is a growing dataset of `(playbook_class, +//! target) -> verdict`. This turns that history into a *prior*: given a +//! target, rank the playbook classes by how likely each is to yield a +//! shippable win — learned from evidence, not the fixed cheapest-first +//! order alone. +//! +//! The ranking statistic is the **Wilson score lower bound** of the +//! win-rate, not the point estimate. This is the same discipline the rest +//! of the project uses (quote the CI lower bound, never the median): a +//! class that won 1/1 does NOT outrank one that won 8/10, because its +//! interval is wide. With no data a class scores 0 and falls back to the +//! cheapest-first prior. As the ledger grows, the estimates sharpen — the +//! policy is honest about its own uncertainty by construction. + +/// Wilson score interval lower bound for `wins`/`n` at z (1.96 = 95%). +/// Returns 0.0 for n == 0 (no evidence). +pub fn wilson_lower_bound(wins: u64, n: u64, z: f64) -> f64 { + if n == 0 { + return 0.0; + } + let n = n as f64; + let p = wins as f64 / n; + let z2 = z * z; + let denom = 1.0 + z2 / n; + let center = p + z2 / (2.0 * n); + let margin = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt(); + ((center - margin) / denom).max(0.0) +} + +/// Aggregated evidence for one playbook class. +#[derive(Debug, Clone, PartialEq)] +pub struct ClassStat { + pub class: u8, + pub attempts: u64, + /// Shippable wins: `accepted` AND sanitizers-clean. + pub wins: u64, + /// Real wins held for a human ruling (needs-human-review) — counted + /// separately, since they are neither clean auto-wins nor failures. + pub held: u64, + pub win_rate: f64, + pub wilson_lb: f64, +} + +impl ClassStat { + /// A short human note on the evidence state. + pub fn note(&self) -> &'static str { + if self.attempts == 0 { + "untried — cheapest-first prior only" + } else if self.wins > 0 { + "proven winner" + } else if self.held > 0 { + "held for human review (measured win, tier-gated)" + } else { + "evidence against — tried, no shippable win" + } + } +} + +/// Build per-class stats (classes 1..=7) from ledger outcome rows, +/// optionally filtered to one target. Each outcome is +/// `(class, target, verdict, sanitizers_clean)`. +pub fn class_stats( + outcomes: &[(u8, String, String, bool)], + target: Option<&str>, + z: f64, +) -> Vec { + (1u8..=7) + .map(|class| { + let rows = outcomes + .iter() + .filter(|(c, t, _, _)| *c == class && target.is_none_or(|want| t == want)); + let (mut attempts, mut wins, mut held) = (0u64, 0u64, 0u64); + for (_, _, verdict, san) in rows { + attempts += 1; + if verdict == "accepted" && *san { + wins += 1; + } else if verdict == "needs-human-review" { + held += 1; + } + } + let win_rate = if attempts > 0 { + wins as f64 / attempts as f64 + } else { + 0.0 + }; + ClassStat { + class, + attempts, + wins, + held, + win_rate, + wilson_lb: wilson_lower_bound(wins, attempts, z), + } + }) + .collect() +} + +/// Rank classes for selection. Primary key: Wilson lower bound (evidence +/// of success), descending. Ties (notably all the zero-evidence classes) +/// break by: fewer failed attempts first (prefer untried over +/// proven-failure), then cheaper class first (the cost prior). +pub fn rank(mut stats: Vec) -> Vec { + stats.sort_by(|a, b| { + b.wilson_lb + .partial_cmp(&a.wilson_lb) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| { + // failures = attempts without a win or a hold + let fa = a.attempts - a.wins - a.held; + let fb = b.attempts - b.wins - b.held; + fa.cmp(&fb) + }) + .then_with(|| a.class.cmp(&b.class)) + }); + stats +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wilson_is_zero_without_evidence() { + assert_eq!(wilson_lower_bound(0, 0, 1.96), 0.0); + } + + #[test] + fn wilson_penalizes_small_samples() { + // 1/1 must NOT outrank 8/10 — the small sample's interval is wide. + let one_of_one = wilson_lower_bound(1, 1, 1.96); + let eight_of_ten = wilson_lower_bound(8, 10, 1.96); + assert!( + one_of_one < eight_of_ten, + "1/1 ({one_of_one:.3}) should rank below 8/10 ({eight_of_ten:.3})" + ); + } + + #[test] + fn wilson_monotonic_in_evidence() { + // Same rate, more evidence → higher lower bound. + assert!(wilson_lower_bound(50, 100, 1.96) > wilson_lower_bound(5, 10, 1.96)); + } + + #[test] + fn rank_prefers_proven_then_untried_then_failed() { + let outcomes = vec![ + (5u8, "t".into(), "accepted".into(), true), + (5u8, "t".into(), "accepted".into(), true), + (5u8, "t".into(), "rejected-bench".into(), false), + (6u8, "t".into(), "rejected-bench".into(), false), // evidence against + // class 7 untried + ]; + let ranked = rank(class_stats(&outcomes, None, 1.96)); + assert_eq!(ranked[0].class, 5, "proven class 5 ranks first"); + // untried class (7) must rank above the tried-and-failed class 6 + let pos7 = ranked.iter().position(|s| s.class == 7).unwrap(); + let pos6 = ranked.iter().position(|s| s.class == 6).unwrap(); + assert!(pos7 < pos6, "untried class 7 outranks failed class 6"); + } +} diff --git a/crates/policy/src/main.rs b/crates/policy/src/main.rs new file mode 100644 index 0000000..c4eb7e1 --- /dev/null +++ b/crates/policy/src/main.rs @@ -0,0 +1,104 @@ +//! `policy rank` — the learned class-selection prior (SPEC §13). +//! Reads the append-only ledger and ranks playbook classes by the Wilson +//! lower bound of their observed shippable-win rate, degrading to +//! cheapest-first where evidence is thin. Advisory only: the gates still +//! decide every verdict. This just tells the agent where to look first. + +use anyhow::Result; +use clap::Parser; +use policy::{class_stats, rank}; +use std::path::PathBuf; + +const CLASS_NAME: [&str; 8] = [ + "", + "build-config", + "allocator", + "alloc-churn", + "data-layout", + "algorithmic", + "simd", + "concurrency", +]; + +#[derive(Parser)] +#[command( + name = "policy", + about = "Learned playbook-class ranking from the ledger (SPEC §13)" +)] +struct Cli { + /// Restrict the evidence to one target (else pool all targets). + #[arg(long)] + target: Option, + #[arg(long, default_value = "results/ledger.sqlite")] + db: PathBuf, + /// z for the Wilson interval (1.96 = 95%). + #[arg(long, default_value_t = 1.96)] + z: f64, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let ledger = ledger::Ledger::open(&cli.db)?; + let outcomes = ledger.all_outcomes()?; + let ranked = rank(class_stats(&outcomes, cli.target.as_deref(), cli.z)); + + let scope = cli.target.as_deref().unwrap_or("all targets"); + println!( + "learned class-selection ranking — evidence: {} ledger rows, scope: {scope}\n\ + (Wilson 95% lower bound of the shippable-win rate; ties break untried-before-failed, then cheapest-first)\n", + outcomes.len() + ); + println!( + "{:>4} {:<13} {:>8} {:>5} {:>5} {:>9} {:>10} note", + "rank", "class", "attempts", "wins", "held", "win-rate", "wilson-lb" + ); + println!("{}", "-".repeat(78)); + for (i, s) in ranked.iter().enumerate() { + println!( + "{:>4} {:>1} {:<11} {:>8} {:>5} {:>5} {:>8.0}% {:>10.3} {}", + i + 1, + s.class, + CLASS_NAME[s.class as usize], + s.attempts, + s.wins, + s.held, + s.win_rate * 100.0, + s.wilson_lb, + s.note(), + ); + } + + // The actionable line: the first class with real supporting evidence, + // else the cheapest untried class. + let best_evidenced = ranked.iter().find(|s| s.wilson_lb > 0.0); + let cheapest_untried = ranked + .iter() + .filter(|s| s.attempts == 0) + .min_by_key(|s| s.class); + println!(); + match (best_evidenced, cheapest_untried) { + (Some(e), _) => println!( + "recommendation: try class {} ({}) first — best evidence (Wilson lb {:.3}); \ + then the next-ranked class whose profile preconditions match.", + e.class, CLASS_NAME[e.class as usize], e.wilson_lb + ), + (None, Some(u)) => println!( + "recommendation: no class has supporting evidence yet; fall back to the \ + cheapest untried class {} ({}).", + u.class, CLASS_NAME[u.class as usize] + ), + (None, None) => { + println!("recommendation: no evidence and no untried class; use cheapest-first.") + } + } + println!( + "\nAdvisory only — the equivalence + significance gates decide every verdict. \ + This ranking sharpens as the ledger grows." + ); + println!( + "note: a 'win' counts only machine-sanitizer-verified accepts. Pre-sanitizer-gate \ + accepts and the overturned comrak-010 are conservatively excluded — the policy \ + trusts the ledger's machine record, not the external audit narrative." + ); + Ok(()) +} diff --git a/justfile b/justfile index 20d252c..689e5b5 100644 --- a/justfile +++ b/justfile @@ -84,3 +84,7 @@ service-calibrate server doc out *flags: # dollars and/or latency percentiles, CIs + methodology printed inline. report run_id *flags: cargo run -p report -- --run-id "{{run_id}}" {{flags}} + +# Learned playbook-class ranking from the ledger (SPEC §13 research fork) +policy *flags: + cargo run -p policy -- {{flags}}