mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
Bootstrap hotpath from engineering spec v0.1
Commit the spec and scaffold the system it describes:
- SPEC.md: the full v0.1 engineering spec
- crates/: trust-layer Rust workspace, unit-tested
- bench-runner: interleaved ABAB scheduling with discarded warm-ups,
deterministic bootstrap CIs over the ratio of medians,
CI-lower-bound accept rule, env fingerprinting, and a CLI with
compare + A/A self-test modes (A/A verified passing end-to-end)
- diff-test: gate outcome model, sha256 corpus pin verifier that
refuses on mismatch, per-target equivalence policy parser
(byte-identical default, explicit fp-tolerance opt-in)
- ledger: append-only SQLite attempt schema with UPDATE/DELETE
refused by triggers, anti-double-attempt query
- report: cores-saved and dollars/yr ROI math over speedup CIs,
pricing.toml inputs
- playbook/: classes 1-7 with preconditions, procedure, verification
notes, and known failure modes per class
- agent/: Python package skeleton — six-tool contract and prompting
spine, harness IPC deferred to Phase 1
- config/: accept.toml (2% threshold, 30 runs/side), pricing.toml
- targets/, corpora/, results/calibration/: layout and policy docs
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
use anyhow::Result;
|
||||
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
||||
use clap::{Parser, Subcommand};
|
||||
use ledger::Verdict;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")]
|
||||
struct Cli {
|
||||
/// Path to accept.toml; built-in defaults if omitted.
|
||||
#[arg(long)]
|
||||
config: Option<PathBuf>,
|
||||
#[command(subcommand)]
|
||||
command: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Interleaved A/B comparison of two shell commands.
|
||||
Compare {
|
||||
#[arg(long)]
|
||||
baseline: String,
|
||||
#[arg(long)]
|
||||
candidate: String,
|
||||
},
|
||||
/// A/A self-test: same command both sides; must yield a null verdict.
|
||||
Aa {
|
||||
#[arg(long)]
|
||||
cmd: String,
|
||||
},
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let cfg = match &cli.config {
|
||||
Some(p) => AcceptConfig::load(p)?,
|
||||
None => AcceptConfig::default(),
|
||||
};
|
||||
|
||||
let (baseline, candidate, aa_mode) = match &cli.command {
|
||||
Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
||||
};
|
||||
|
||||
let fp = EnvFingerprint::collect("none", "system-default");
|
||||
eprintln!("env fingerprint: {}", serde_json::to_string(&fp)?);
|
||||
eprintln!(
|
||||
"plan: {} measured + {} warm-up runs per side, interleaved",
|
||||
cfg.runs_per_side, cfg.warmup_runs
|
||||
);
|
||||
|
||||
let samples =
|
||||
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs)?;
|
||||
let ci = stats::bootstrap_ratio_ci(
|
||||
&samples.baseline_s,
|
||||
&samples.candidate_s,
|
||||
cfg.bootstrap_iters,
|
||||
cfg.confidence,
|
||||
cfg.bootstrap_seed,
|
||||
);
|
||||
|
||||
println!(
|
||||
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}]",
|
||||
ci.median,
|
||||
cfg.confidence * 100.0,
|
||||
ci.lo,
|
||||
ci.hi
|
||||
);
|
||||
|
||||
if aa_mode {
|
||||
// Null verdict required: an "accept" here is a calibration failure.
|
||||
let verdict = decide(ci, cfg.threshold);
|
||||
if verdict == Verdict::Accepted {
|
||||
anyhow::bail!("A/A self-test FAILED: harness claims a speedup from identical binaries");
|
||||
}
|
||||
println!("A/A self-test passed (null verdict)");
|
||||
} else {
|
||||
println!("verdict: {}", decide(ci, cfg.threshold).as_str());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user