mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
phase5: learned class-selection policy (SPEC §13) — rank classes by ledger evidence
crates/policy: reads the append-only ledger and ranks playbook classes by the Wilson score lower bound of their machine-verified win rate — the same 'quote the lower bound' discipline the rest of the project uses. A class that won 1/1 does not outrank one that won 8/10 (wider interval); untried classes fall back to the cheapest-first prior; tried-and-failed classes rank last. Advisory only — the gates still decide every verdict. On the current 33-row ledger it ranks class 5 (algorithmic) first, then class 3, then class 1 — the proven winners — with class 7 (untried) above the evidence-against classes. Wins count only machine-sanitizer-verified accepts (conservatively excluding pre-gate accepts and the overturned comrak-010): the policy trusts the ledger's machine record, not narrative. 4 unit tests (Wilson monotonicity, small-sample penalty, ranking order). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
Claude
@@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "policy"
|
||||
description = "Learned class-selection policy (SPEC §13): rank playbook classes by the Wilson lower bound of their observed win-rate in the ledger, degrading to cheapest-first when data is thin"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
ledger = { path = "../ledger" }
|
||||
@@ -0,0 +1,162 @@
|
||||
//! Learned class-selection policy (SPEC §13 research fork).
|
||||
//!
|
||||
//! The append-only ledger is a growing dataset of `(playbook_class,
|
||||
//! target) -> verdict`. This turns that history into a *prior*: given a
|
||||
//! target, rank the playbook classes by how likely each is to yield a
|
||||
//! shippable win — learned from evidence, not the fixed cheapest-first
|
||||
//! order alone.
|
||||
//!
|
||||
//! The ranking statistic is the **Wilson score lower bound** of the
|
||||
//! win-rate, not the point estimate. This is the same discipline the rest
|
||||
//! of the project uses (quote the CI lower bound, never the median): a
|
||||
//! class that won 1/1 does NOT outrank one that won 8/10, because its
|
||||
//! interval is wide. With no data a class scores 0 and falls back to the
|
||||
//! cheapest-first prior. As the ledger grows, the estimates sharpen — the
|
||||
//! policy is honest about its own uncertainty by construction.
|
||||
|
||||
/// Wilson score interval lower bound for `wins`/`n` at z (1.96 = 95%).
|
||||
/// Returns 0.0 for n == 0 (no evidence).
|
||||
pub fn wilson_lower_bound(wins: u64, n: u64, z: f64) -> f64 {
|
||||
if n == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let n = n as f64;
|
||||
let p = wins as f64 / n;
|
||||
let z2 = z * z;
|
||||
let denom = 1.0 + z2 / n;
|
||||
let center = p + z2 / (2.0 * n);
|
||||
let margin = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt();
|
||||
((center - margin) / denom).max(0.0)
|
||||
}
|
||||
|
||||
/// Aggregated evidence for one playbook class.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ClassStat {
|
||||
pub class: u8,
|
||||
pub attempts: u64,
|
||||
/// Shippable wins: `accepted` AND sanitizers-clean.
|
||||
pub wins: u64,
|
||||
/// Real wins held for a human ruling (needs-human-review) — counted
|
||||
/// separately, since they are neither clean auto-wins nor failures.
|
||||
pub held: u64,
|
||||
pub win_rate: f64,
|
||||
pub wilson_lb: f64,
|
||||
}
|
||||
|
||||
impl ClassStat {
|
||||
/// A short human note on the evidence state.
|
||||
pub fn note(&self) -> &'static str {
|
||||
if self.attempts == 0 {
|
||||
"untried — cheapest-first prior only"
|
||||
} else if self.wins > 0 {
|
||||
"proven winner"
|
||||
} else if self.held > 0 {
|
||||
"held for human review (measured win, tier-gated)"
|
||||
} else {
|
||||
"evidence against — tried, no shippable win"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build per-class stats (classes 1..=7) from ledger outcome rows,
|
||||
/// optionally filtered to one target. Each outcome is
|
||||
/// `(class, target, verdict, sanitizers_clean)`.
|
||||
pub fn class_stats(
|
||||
outcomes: &[(u8, String, String, bool)],
|
||||
target: Option<&str>,
|
||||
z: f64,
|
||||
) -> Vec<ClassStat> {
|
||||
(1u8..=7)
|
||||
.map(|class| {
|
||||
let rows = outcomes
|
||||
.iter()
|
||||
.filter(|(c, t, _, _)| *c == class && target.is_none_or(|want| t == want));
|
||||
let (mut attempts, mut wins, mut held) = (0u64, 0u64, 0u64);
|
||||
for (_, _, verdict, san) in rows {
|
||||
attempts += 1;
|
||||
if verdict == "accepted" && *san {
|
||||
wins += 1;
|
||||
} else if verdict == "needs-human-review" {
|
||||
held += 1;
|
||||
}
|
||||
}
|
||||
let win_rate = if attempts > 0 {
|
||||
wins as f64 / attempts as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
ClassStat {
|
||||
class,
|
||||
attempts,
|
||||
wins,
|
||||
held,
|
||||
win_rate,
|
||||
wilson_lb: wilson_lower_bound(wins, attempts, z),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Rank classes for selection. Primary key: Wilson lower bound (evidence
|
||||
/// of success), descending. Ties (notably all the zero-evidence classes)
|
||||
/// break by: fewer failed attempts first (prefer untried over
|
||||
/// proven-failure), then cheaper class first (the cost prior).
|
||||
pub fn rank(mut stats: Vec<ClassStat>) -> Vec<ClassStat> {
|
||||
stats.sort_by(|a, b| {
|
||||
b.wilson_lb
|
||||
.partial_cmp(&a.wilson_lb)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| {
|
||||
// failures = attempts without a win or a hold
|
||||
let fa = a.attempts - a.wins - a.held;
|
||||
let fb = b.attempts - b.wins - b.held;
|
||||
fa.cmp(&fb)
|
||||
})
|
||||
.then_with(|| a.class.cmp(&b.class))
|
||||
});
|
||||
stats
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn wilson_is_zero_without_evidence() {
|
||||
assert_eq!(wilson_lower_bound(0, 0, 1.96), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wilson_penalizes_small_samples() {
|
||||
// 1/1 must NOT outrank 8/10 — the small sample's interval is wide.
|
||||
let one_of_one = wilson_lower_bound(1, 1, 1.96);
|
||||
let eight_of_ten = wilson_lower_bound(8, 10, 1.96);
|
||||
assert!(
|
||||
one_of_one < eight_of_ten,
|
||||
"1/1 ({one_of_one:.3}) should rank below 8/10 ({eight_of_ten:.3})"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wilson_monotonic_in_evidence() {
|
||||
// Same rate, more evidence → higher lower bound.
|
||||
assert!(wilson_lower_bound(50, 100, 1.96) > wilson_lower_bound(5, 10, 1.96));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_prefers_proven_then_untried_then_failed() {
|
||||
let outcomes = vec![
|
||||
(5u8, "t".into(), "accepted".into(), true),
|
||||
(5u8, "t".into(), "accepted".into(), true),
|
||||
(5u8, "t".into(), "rejected-bench".into(), false),
|
||||
(6u8, "t".into(), "rejected-bench".into(), false), // evidence against
|
||||
// class 7 untried
|
||||
];
|
||||
let ranked = rank(class_stats(&outcomes, None, 1.96));
|
||||
assert_eq!(ranked[0].class, 5, "proven class 5 ranks first");
|
||||
// untried class (7) must rank above the tried-and-failed class 6
|
||||
let pos7 = ranked.iter().position(|s| s.class == 7).unwrap();
|
||||
let pos6 = ranked.iter().position(|s| s.class == 6).unwrap();
|
||||
assert!(pos7 < pos6, "untried class 7 outranks failed class 6");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//! `policy rank` — the learned class-selection prior (SPEC §13).
|
||||
//! Reads the append-only ledger and ranks playbook classes by the Wilson
|
||||
//! lower bound of their observed shippable-win rate, degrading to
|
||||
//! cheapest-first where evidence is thin. Advisory only: the gates still
|
||||
//! decide every verdict. This just tells the agent where to look first.
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use policy::{class_stats, rank};
|
||||
use std::path::PathBuf;
|
||||
|
||||
const CLASS_NAME: [&str; 8] = [
|
||||
"",
|
||||
"build-config",
|
||||
"allocator",
|
||||
"alloc-churn",
|
||||
"data-layout",
|
||||
"algorithmic",
|
||||
"simd",
|
||||
"concurrency",
|
||||
];
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "policy",
|
||||
about = "Learned playbook-class ranking from the ledger (SPEC §13)"
|
||||
)]
|
||||
struct Cli {
|
||||
/// Restrict the evidence to one target (else pool all targets).
|
||||
#[arg(long)]
|
||||
target: Option<String>,
|
||||
#[arg(long, default_value = "results/ledger.sqlite")]
|
||||
db: PathBuf,
|
||||
/// z for the Wilson interval (1.96 = 95%).
|
||||
#[arg(long, default_value_t = 1.96)]
|
||||
z: f64,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let ledger = ledger::Ledger::open(&cli.db)?;
|
||||
let outcomes = ledger.all_outcomes()?;
|
||||
let ranked = rank(class_stats(&outcomes, cli.target.as_deref(), cli.z));
|
||||
|
||||
let scope = cli.target.as_deref().unwrap_or("all targets");
|
||||
println!(
|
||||
"learned class-selection ranking — evidence: {} ledger rows, scope: {scope}\n\
|
||||
(Wilson 95% lower bound of the shippable-win rate; ties break untried-before-failed, then cheapest-first)\n",
|
||||
outcomes.len()
|
||||
);
|
||||
println!(
|
||||
"{:>4} {:<13} {:>8} {:>5} {:>5} {:>9} {:>10} note",
|
||||
"rank", "class", "attempts", "wins", "held", "win-rate", "wilson-lb"
|
||||
);
|
||||
println!("{}", "-".repeat(78));
|
||||
for (i, s) in ranked.iter().enumerate() {
|
||||
println!(
|
||||
"{:>4} {:>1} {:<11} {:>8} {:>5} {:>5} {:>8.0}% {:>10.3} {}",
|
||||
i + 1,
|
||||
s.class,
|
||||
CLASS_NAME[s.class as usize],
|
||||
s.attempts,
|
||||
s.wins,
|
||||
s.held,
|
||||
s.win_rate * 100.0,
|
||||
s.wilson_lb,
|
||||
s.note(),
|
||||
);
|
||||
}
|
||||
|
||||
// The actionable line: the first class with real supporting evidence,
|
||||
// else the cheapest untried class.
|
||||
let best_evidenced = ranked.iter().find(|s| s.wilson_lb > 0.0);
|
||||
let cheapest_untried = ranked
|
||||
.iter()
|
||||
.filter(|s| s.attempts == 0)
|
||||
.min_by_key(|s| s.class);
|
||||
println!();
|
||||
match (best_evidenced, cheapest_untried) {
|
||||
(Some(e), _) => println!(
|
||||
"recommendation: try class {} ({}) first — best evidence (Wilson lb {:.3}); \
|
||||
then the next-ranked class whose profile preconditions match.",
|
||||
e.class, CLASS_NAME[e.class as usize], e.wilson_lb
|
||||
),
|
||||
(None, Some(u)) => println!(
|
||||
"recommendation: no class has supporting evidence yet; fall back to the \
|
||||
cheapest untried class {} ({}).",
|
||||
u.class, CLASS_NAME[u.class as usize]
|
||||
),
|
||||
(None, None) => {
|
||||
println!("recommendation: no evidence and no untried class; use cheapest-first.")
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"\nAdvisory only — the equivalence + significance gates decide every verdict. \
|
||||
This ranking sharpens as the ledger grows."
|
||||
);
|
||||
println!(
|
||||
"note: a 'win' counts only machine-sanitizer-verified accepts. Pre-sanitizer-gate \
|
||||
accepts and the overturned comrak-010 are conservatively excluded — the policy \
|
||||
trusts the ledger's machine record, not the external audit narrative."
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user