mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
Bootstrap hotpath from engineering spec v0.1
Commit the spec and scaffold the system it describes:
- SPEC.md: the full v0.1 engineering spec
- crates/: trust-layer Rust workspace, unit-tested
- bench-runner: interleaved ABAB scheduling with discarded warm-ups,
deterministic bootstrap CIs over the ratio of medians,
CI-lower-bound accept rule, env fingerprinting, and a CLI with
compare + A/A self-test modes (A/A verified passing end-to-end)
- diff-test: gate outcome model, sha256 corpus pin verifier that
refuses on mismatch, per-target equivalence policy parser
(byte-identical default, explicit fp-tolerance opt-in)
- ledger: append-only SQLite attempt schema with UPDATE/DELETE
refused by triggers, anti-double-attempt query
- report: cores-saved and dollars/yr ROI math over speedup CIs,
pricing.toml inputs
- playbook/: classes 1-7 with preconditions, procedure, verification
notes, and known failure modes per class
- agent/: Python package skeleton — six-tool contract and prompting
spine, harness IPC deferred to Phase 1
- config/: accept.toml (2% threshold, 30 runs/side), pricing.toml
- targets/, corpora/, results/calibration/: layout and policy docs
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
Claude
@@ -216,3 +216,11 @@ __marimo__/
|
||||
|
||||
# Streamlit
|
||||
.streamlit/secrets.toml
|
||||
|
||||
# Rust (trust-layer workspace)
|
||||
target/
|
||||
|
||||
# hotpath run outputs
|
||||
results/**/*.json
|
||||
results/**/*.sqlite
|
||||
targets/*/workspace/
|
||||
|
||||
Generated
+614
@@ -0,0 +1,614 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.103"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
|
||||
|
||||
[[package]]
|
||||
name = "bench-runner"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"ledger",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.66"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diff-test"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
|
||||
dependencies = [
|
||||
"hashbrown 0.14.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "ledger"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "report"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"ledger",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.32.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "0.6.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.118"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.8.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_spanned",
|
||||
"toml_datetime",
|
||||
"toml_edit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.6.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.22.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"serde",
|
||||
"serde_spanned",
|
||||
"toml_datetime",
|
||||
"toml_write",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_write"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/bench-runner",
|
||||
"crates/diff-test",
|
||||
"crates/ledger",
|
||||
"crates/report",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "GPL-3.0-or-later"
|
||||
repository = "https://github.com/bdbrown4/peltier.io"
|
||||
|
||||
[workspace.dependencies]
|
||||
anyhow = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
sha2 = "0.10"
|
||||
thiserror = "2"
|
||||
toml = "0.8"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
@@ -1,2 +1,60 @@
|
||||
# peltier.io
|
||||
|
||||
Spot cooling for hot paths. A profile-guided optimization agent that only ships wins it can prove.
|
||||
|
||||
hotpath finds the 10–90% left on the floor between "the compiler" and
|
||||
"hire a perf consultant," with two properties no consultant offers:
|
||||
**verified equivalence** (a change provably preserves behavior or is
|
||||
discarded) and **self-verifying ROI** (the value claim is a stopwatch on
|
||||
trusted infrastructure, not a survey).
|
||||
|
||||
Full design: **[SPEC.md](SPEC.md)**.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
crates/ Trust layer (agent has NO write access — SPEC §10)
|
||||
bench-runner/ Interleaved A/B timing, bootstrap CIs, A/A calibration
|
||||
diff-test/ Equivalence gates, corpus hash-pinning, per-target policy
|
||||
ledger/ Append-only SQLite attempt ledger (enforced by triggers)
|
||||
report/ ROI math: speedup CI → cores → dollars, caveats included
|
||||
agent/ Untrusted proposer (Claude Agent SDK, Python)
|
||||
playbook/ Optimization classes 1–7, tried strictly cheapest-first
|
||||
config/ accept.toml (thresholds), pricing.toml (ROI inputs)
|
||||
targets/ Per-target checkouts — the only agent-writable path
|
||||
corpora/ Hash-pinned golden-replay inputs (read-only to agent)
|
||||
results/ Calibration evidence, per-engagement outputs
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```sh
|
||||
cargo test --workspace # trust-layer unit tests
|
||||
just aa # A/A self-test: must yield a null verdict
|
||||
just compare "cmd-a" "cmd-b" # interleaved A/B with bootstrap CI
|
||||
```
|
||||
|
||||
## Status
|
||||
|
||||
Phase 0–1 scaffold (see SPEC §5 for the phase plan and exit criteria).
|
||||
|
||||
Done:
|
||||
- Workspace with the four trust-layer crates; core statistics
|
||||
(interleaved scheduling, bootstrap ratio CIs, CI-lower-bound accept
|
||||
rule), append-only ledger schema with mutation-refusing triggers,
|
||||
corpus hash-pin verifier, equivalence-policy parser, ROI formulas —
|
||||
all unit-tested.
|
||||
- bench-runner CLI: whole-program `compare` and `aa` modes with env
|
||||
fingerprinting.
|
||||
- Playbook classes 1–7; agent tool contract and prompting spine.
|
||||
|
||||
Next (Phase 1 exit criteria):
|
||||
- `perf stat` counters + max RSS + RAPL capture per run.
|
||||
- Automated A/A calibration sessions and regression-injection self-test
|
||||
recorded to `results/calibration/` (<5% FP, ≥95% detection).
|
||||
- diff-test orchestration end-to-end on two real targets; harness IPC
|
||||
for the agent tool surface.
|
||||
|
||||
## License
|
||||
|
||||
GPL-3.0-or-later — see [LICENSE](LICENSE).
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# hotpath — Engineering Spec v0.1
|
||||
|
||||
## 1. Mission & thesis
|
||||
|
||||
Performance engineering is scarce artisan labor. Compilers closed the gap at general codegen; nothing closed it at the application and infrastructure layer, where profile-guided rewrites, build configuration, data-layout changes, and kernel-level work still leave 10–90% on the floor. hotpath is an agent that harvests that gap with two properties no consultant offers: **verified equivalence** (a change provably preserves behavior or is discarded) and **self-verifying ROI** (the value claim is a stopwatch on trusted infrastructure, not a survey).
|
||||
|
||||
Why this wins:
|
||||
- **Empty niche.** Between "the compiler" and "hire a perf consultant" there is no product. LLM codegen is crowded; verified optimization is not.
|
||||
- **The metric defends itself.** Same inputs, same outputs, fewer cycles, with confidence intervals. No correlation-causation problem is possible by construction.
|
||||
- **The ledger compounds.** Every accepted/rejected attempt with full evidence is future RL training data and, eventually, the substrate for learned optimization policies (see §13, research fork).
|
||||
|
||||
## 2. System overview
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────┐
|
||||
│ TRUST LAYER │
|
||||
│ (agent has NO write access to this box) │
|
||||
┌─────────┐ patch │ ┌────────────┐ ┌───────────┐ ┌───────┐ │
|
||||
│ agent/ │──────────▶│ │ diff-test │─▶│bench-runner│─▶│ledger │ │
|
||||
│ (Agent │ via git │ │ gates │ │ interleaved│ │SQLite │ │
|
||||
│ SDK) │ apply + │ └────────────┘ │ A/B + CIs │ └───┬───┘ │
|
||||
└────┬────┘ allowlist│ └───────────┘ │ │
|
||||
│ └──────────────────────────────────────┼──────┘
|
||||
│ reads profiles, ledger, playbook ▼
|
||||
▼ ┌─────────┐
|
||||
┌──────────┐ │ report/ │
|
||||
│ profiler │ perf / flamegraph / coz │ ROI │
|
||||
│ adapter │ └─────────┘
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
Loop: profile → rank hotspots → select playbook class → hypothesize → patch → gates → bench → verdict → ledger → repeat.
|
||||
|
||||
## 3. Component specs
|
||||
|
||||
### 3.1 `bench-runner` (the product is this crate)
|
||||
|
||||
Requirements:
|
||||
- Runs baseline and candidate **interleaved** (ABABAB…), never sequential blocks, to control thermal and background drift.
|
||||
- Baseline is rebuilt from a **pristine checkout** each session — never from agent workspace.
|
||||
- Environment control: CPU pinning (`taskset`/cgroup cpuset), performance governor, turbo/SMT state recorded (disabled where root permits), ASLR handled explicitly (see §7), no-network container, warm-up runs discarded.
|
||||
- Statistics: ≥30 measured runs per side (configurable); report median and bootstrap 95% CI of the ratio; effect accepted only if CI lower bound ≥ threshold (`config/accept.toml`, default 2%).
|
||||
- **A/A self-test mode**: run the same binary as both sides; must produce a null verdict. Scheduled automatically before any measurement session.
|
||||
- Regression-injection self-test: a synthetic 5% slowdown must be detected ≥95% of the time.
|
||||
- Metrics captured per run: wall time, cycles, instructions, cache misses, branch misses (via `perf stat`), max RSS, and RAPL energy where available.
|
||||
- Modes: (a) whole-program via hyperfine-style CLI timing; (b) microbench via criterion harness hooks; (c) service mode (Phase 4): wrk2 / vegeta load replay with **coordinated-omission-aware** latency recording (HdrHistogram).
|
||||
|
||||
Acceptance: passes A/A calibration at <5% false-positive rate and injected-regression detection at ≥95%, documented in `results/calibration/`.
|
||||
|
||||
### 3.2 `diff-test` (equivalence gates)
|
||||
|
||||
Layered, all must pass:
|
||||
1. **Upstream test suite** of the target, unmodified, green.
|
||||
2. **Golden replay**: recorded input corpus → byte-identical outputs (stdout, files, exit codes). FP-producing targets declare an explicit tolerance policy in `targets/<name>/equivalence.toml`; absent a policy, byte-identical is required.
|
||||
3. **Differential fuzzing** of changed functions: old vs. new implementation compiled side by side, driven by proptest/cargo-fuzz (Rust) or libFuzzer (C/C++), outputs compared. Minimum 60s or 10k iterations per changed function.
|
||||
4. **Sanitizers** on the candidate: ASan + UBSan always; TSan when the diff touches threads, atomics, or locks; MIRI for Rust targets where feasible.
|
||||
|
||||
Corpus and test files live outside agent-writable paths and are hash-pinned; `diff-test` refuses to run if hashes mismatch.
|
||||
|
||||
### 3.3 `ledger`
|
||||
|
||||
Append-only SQLite. One row per attempt:
|
||||
`run_id, timestamp, target, target_commit, phase, hotspot (symbol + % of profile), playbook_class, hypothesis, patch (diff), gates {tests, golden, fuzz_iters, sanitizers}, bench {baseline_ci, candidate_ci, delta_ci, env_fingerprint}, verdict {accepted | rejected-gate | rejected-bench | needs-human-review}, tokens_spent, wall_time`.
|
||||
|
||||
Nothing is ever deleted. The ledger is the audit trail, the anti-double-attempt memory, and the future training set.
|
||||
|
||||
### 3.4 `profiler adapter`
|
||||
|
||||
Phase 0–2: `perf record` + flamegraphs (inferno) + `perf stat`; hotspot ranking = exclusive samples by symbol with source mapping. Phase 3: add **coz** causal profiling — "what does speeding this line up buy end-to-end" — which is the literal bridge from cycles to ROI and the correct prioritizer for multi-threaded targets.
|
||||
|
||||
### 3.5 `agent/`
|
||||
|
||||
Claude Agent SDK (Python). Tools exposed to the model: `read_profile`, `read_ledger`, `read_playbook`, `propose_patch(diff, hypothesis)`, `run_verdict`, `read_target_source` (read-only). Deliberately **not** exposed: shell on the host, writes outside `targets/<name>/`, any access to `crates/`, `config/`, `corpora/`.
|
||||
|
||||
Prompting spine: (1) state hypothesis before patching; (2) prefer the cheapest untried playbook class; (3) max two iterations per rejected hypothesis; (4) a rejection with a clean ledger row is a successful outcome.
|
||||
|
||||
### 3.6 `report/`
|
||||
|
||||
Consumes ledger rows, emits per-engagement ROI report:
|
||||
- Throughput jobs: cores saved = fleet_cores × (1 − 1/speedup); $ = cores × $/core-hr × hours/yr (rates in `config/pricing.toml`).
|
||||
- Services: p50/p95/p99 deltas under replayed traffic, with the business-value mapping left as a customer-supplied parameter and one worked example.
|
||||
- Optional: joules saved (RAPL) for the sustainability line.
|
||||
- Every figure carries CI + workload description + environment fingerprint. Caveats are printed on the report, not in an appendix.
|
||||
|
||||
## 4. Optimization playbook v0 (strict order)
|
||||
|
||||
1. **Build configuration**: opt level, `-march`/`target-cpu`, LTO (thin → fat), PGO, BOLT where applicable.
|
||||
2. **Allocator swap**: mimalloc/jemalloc trial. (Classes 1–2 are the "your 30% was a linker flag" tier — always exhausted first.)
|
||||
3. **Allocation churn**: hoisting, reuse, arenas, small-vector.
|
||||
4. **Data layout**: struct packing/reordering, AoS→SoA, false-sharing fixes, cache blocking.
|
||||
5. **Algorithmic**: complexity class, better std/library primitive, memoization, precomputation.
|
||||
6. **SIMD**: autovectorization enablement first (aliasing, bounds, FP flags — FP flags trigger human review), intrinsics last.
|
||||
7. **Concurrency**: contention reduction, sharding, lock-free only with `needs-human-review`.
|
||||
|
||||
Each class ships as `playbook/NN-name.md`: preconditions (profile signature that suggests it), procedure, verification notes, known failure modes.
|
||||
|
||||
## 5. Phase plan & exit criteria
|
||||
|
||||
| Phase | Scope | Exit criteria |
|
||||
|---|---|---|
|
||||
| **0** Manual dry run (1–2 wknd) | Two CPU-bound OSS targets; loop run by hand with Claude Code as copilot | ≥1 verified win ≥10% on one target; written case study with methodology |
|
||||
| **1** Trust layer (3–4 wknd) | bench-runner + diff-test + ledger | A/A false-positive <5%; injected 5% regression caught ≥95%; gates run on 2 targets end-to-end |
|
||||
| **2** Agent (3–4 wknd) | Agent SDK loop, Rust targets first | Unattended profile→verdict loop on 1 target; ≥1 auto-accepted win; **zero false accepts** across ≥20 audited attempts |
|
||||
| **3** Playbook + proof | Classes 3–7, coz, C/C++ targets | 3–5 public case studies; zero regressions shipped; playbook ≥6 classes |
|
||||
| **4** Services & scale | Load replay, latency mode, ROI reports | One real service workload optimized under replayed traffic; report generated mechanically |
|
||||
|
||||
Language order: Rust first (criterion/proptest/MIRI make Phases 1–2 cheaper), C/C++ second (the richer legacy hunting ground), services third.
|
||||
|
||||
## 6. Target selection criteria (Phase 0–3)
|
||||
|
||||
CPU-bound (>70% user time on profile); meaningful test suite; existing benchmark or easily scripted workload; permissive license; active but not hyper-optimized (skip anything with a dedicated perf team); builds cleanly in container. Good hunting: codecs, parsers, serializers, image processing, compression, static-site generators, linters.
|
||||
|
||||
## 7. Statistical methodology notes
|
||||
|
||||
- Interleaved A/B with pristine-checkout baselines is the core defense against drift.
|
||||
- **Layout bias** (Mytkowicz et al., ASPLOS'09: link order and env size alone swing results past "real" effects): mitigate by randomizing link order / ASLR *across* runs and aggregating, Stabilizer-style, rather than fixing one lucky layout. Document the chosen mode in the env fingerprint.
|
||||
- Report ratios with bootstrap CIs; never means without spread; never a single run, ever.
|
||||
- Service latency: wrk2 constant-throughput mode + HdrHistogram to avoid coordinated omission; never trust closed-loop load generators for p99 claims.
|
||||
|
||||
## 8. Equivalence policy tiers
|
||||
|
||||
- **Auto-acceptable**: bit-identical outputs across all gates, no FP-ordering changes, no concurrency-primitive changes, sanitizers clean.
|
||||
- **needs-human-review** (never auto-accepted): FP reassociation/contraction or `-ffast-math`-class flags; any diff touching atomics, locks, memory ordering, or thread counts; unsafe blocks / raw pointer arithmetic introduced; anything the sanitizers flag even as warnings.
|
||||
- **Auto-rejected**: any gate failure, any divergence under differential fuzz, patch outside allowlist.
|
||||
|
||||
## 9. ROI methodology (the pitch artifact)
|
||||
|
||||
The report answers one question: *what did the stopwatch say, and what does that cost or buy?* Throughput → cores → dollars via public cloud pricing; latency → percentile deltas under recorded production-shaped load, priced by the customer's own latency-value number; batch windows → wall-clock hours returned to the schedule. All three come with CIs and a workload statement. The methodology section ships *inside* every report so the number survives hostile review — that is the differentiation.
|
||||
|
||||
## 10. Security & anti-reward-hacking (load-bearing)
|
||||
|
||||
The agent's incentive is "make the number go up"; the design must make cheating structurally impossible, not merely discouraged:
|
||||
- Trust layer (`crates/`, `config/`, `corpora/`, upstream tests) is **read-only to the agent** — enforced by filesystem permissions and a separate-process harness, not by prompt.
|
||||
- Patches are applied by the harness via `git apply` after a path-allowlist check; the agent never holds a general shell on the host.
|
||||
- Baselines rebuilt from pristine checkouts; corpora hash-pinned; `diff-test` refuses on mismatch.
|
||||
- Target code executes only in no-network containers (bench containers additionally seccomp-restricted).
|
||||
- Periodic human audit of accepted wins (100% in Phase 2, sampled thereafter); any false accept is a stop-the-line event.
|
||||
|
||||
## 11. Risk register
|
||||
|
||||
| Risk | Mitigation |
|
||||
|---|---|
|
||||
| Measurement bias / noise | §7 discipline; A/A tests before every session; env fingerprints |
|
||||
| Overfitting to the benchmark | Holdout workloads; Phase 4 uses recorded real traffic only |
|
||||
| FP equivalence gaps | Explicit per-target tolerance policy; default byte-identical; human review tier |
|
||||
| Concurrency bugs introduced | TSan gate; human-review tier for all concurrency diffs |
|
||||
| UB introduced | UBSan always; MIRI for Rust; human-review for unsafe |
|
||||
| Agent games the harness | §10 structural controls; audits |
|
||||
| License/IP hygiene | Permissive-license targets only for public case studies |
|
||||
| Wins too small to matter | Cheap-wins-first ordering keeps cost-per-attempt low; ledger prevents re-grinding dead ends |
|
||||
|
||||
## 12. Tech stack
|
||||
|
||||
Rust workspace (`bench-runner`, `diff-test`, `ledger`, `report`) · `just` · hyperfine-style timing + criterion hooks · perf, inferno flamegraphs, coz · proptest, cargo-fuzz, libFuzzer · ASan/UBSan/TSan, MIRI · podman/docker (no-net) · wrk2 + HdrHistogram, vegeta · SQLite · Python 3.11+ with Claude Agent SDK (`agent/`) — confirm current SDK package name and auth at https://docs.claude.com/en/docs/claude-code/overview.
|
||||
|
||||
## 13. Later / research fork (explicitly out of scope now)
|
||||
|
||||
- GPU kernel lane (where 2–10x gaps live): Triton/CUDA targets, correctness via reference-kernel differential testing, reward = measured kernel time.
|
||||
- Learned optimization policies: ledger → RL; DELTA-over-program-graphs (ProGraML-style IR graphs) as the policy/ranking model. Prior art anchors: MLGO (learned inlining/regalloc in LLVM), AlphaDev (Nature 2023), Meta LLM Compiler (2024).
|
||||
|
||||
## 14. References
|
||||
|
||||
Mytkowicz et al., *Producing Wrong Data Without Doing Anything Obviously Wrong* (ASPLOS 2009) — read before building bench-runner · Curtsinger & Berger, *Stabilizer* and *coz* · Gil Tene on coordinated omission · MLGO (Trofin et al.) · ProGraML (Cummins et al.) · AlphaDev (Mankowitz et al., Nature 2023) · Meta LLM Compiler (Cummins et al., 2024).
|
||||
@@ -0,0 +1,10 @@
|
||||
"""hotpath agent (SPEC §3.5).
|
||||
|
||||
The agent is the *untrusted* half of the system. It reads profiles, the
|
||||
ledger, and the playbook; it proposes patches and requests verdicts. It
|
||||
never holds a shell on the host, never writes outside targets/<name>/,
|
||||
and never touches crates/, config/, or corpora/ — enforced by the
|
||||
harness and filesystem permissions, not by this code or any prompt.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Prompting spine (SPEC §3.5). The system prompt is versioned code, not
|
||||
a config knob: changes here change what the ledger's attempts mean."""
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You are hotpath, a performance-optimization agent. You operate in a loop:
|
||||
profile -> rank hotspots -> select playbook class -> hypothesize -> patch
|
||||
-> gates -> bench -> verdict -> ledger.
|
||||
|
||||
Rules, in priority order:
|
||||
|
||||
1. State your hypothesis BEFORE proposing a patch: which hotspot, which
|
||||
playbook class, what mechanism, and what speedup you expect. The
|
||||
hypothesis is recorded verbatim in the ledger.
|
||||
|
||||
2. Prefer the cheapest untried playbook class for the current hotspot.
|
||||
Check the ledger first; never re-attempt a (hotspot, class,
|
||||
hypothesis) combination that already has a verdict.
|
||||
|
||||
3. At most two iterations per rejected hypothesis. After the second
|
||||
rejection, move to the next class or the next hotspot.
|
||||
|
||||
4. A rejection with a clean ledger row is a successful outcome. You are
|
||||
measured on the integrity of the ledger, not on acceptance rate.
|
||||
Never restructure a patch to influence measurement rather than to
|
||||
express the hypothesis.
|
||||
|
||||
5. Equivalence is not yours to relax. Floating-point flag changes,
|
||||
concurrency-primitive changes, and new unsafe code are routed to
|
||||
human review automatically; do not attempt to phrase a patch to
|
||||
avoid that routing.
|
||||
"""
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Tool surface exposed to the model (SPEC §3.5).
|
||||
|
||||
Exactly six tools, all mediated by the trust-layer harness over IPC —
|
||||
the agent process has no direct filesystem or shell capability beyond
|
||||
what these return. Deliberately NOT exposed: shell on the host, writes
|
||||
outside targets/<name>/, any access to crates/, config/, corpora/.
|
||||
|
||||
Phase 2 wiring: each function below becomes a Claude Agent SDK tool
|
||||
definition whose implementation calls the harness. The signatures and
|
||||
docstrings are the contract; the bodies are placeholders until the
|
||||
harness IPC exists (Phase 1 exit criteria).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
class HarnessUnavailable(RuntimeError):
|
||||
"""Raised until the Phase 1 trust-layer harness IPC is wired up."""
|
||||
|
||||
|
||||
def read_profile(target: str) -> dict[str, Any]:
|
||||
"""Latest profile for a target: ranked hotspots (symbol, exclusive %,
|
||||
source mapping) plus perf stat counters. Read-only."""
|
||||
raise HarnessUnavailable("profiler adapter not wired (Phase 1)")
|
||||
|
||||
|
||||
def read_ledger(target: str) -> list[dict[str, Any]]:
|
||||
"""All prior attempts against this target — the anti-double-attempt
|
||||
memory. The agent must not re-grind a (hotspot, playbook_class,
|
||||
hypothesis) combination that already has a verdict."""
|
||||
raise HarnessUnavailable("ledger IPC not wired (Phase 1)")
|
||||
|
||||
|
||||
def read_playbook(class_number: int) -> str:
|
||||
"""Playbook class markdown (1-7): preconditions, procedure,
|
||||
verification notes, known failure modes."""
|
||||
raise HarnessUnavailable("playbook IPC not wired (Phase 1)")
|
||||
|
||||
|
||||
def read_target_source(target: str, path: str) -> str:
|
||||
"""Read-only view of the target's source tree at the pinned commit."""
|
||||
raise HarnessUnavailable("source IPC not wired (Phase 1)")
|
||||
|
||||
|
||||
def propose_patch(target: str, diff: str, hypothesis: str) -> str:
|
||||
"""Submit a unified diff plus the hypothesis it tests. The harness
|
||||
applies it via `git apply` after a path-allowlist check (SPEC §10);
|
||||
a diff touching anything outside targets/<target>/ is auto-rejected
|
||||
before any gate runs. Returns a patch id."""
|
||||
raise HarnessUnavailable("patch IPC not wired (Phase 1)")
|
||||
|
||||
|
||||
def run_verdict(patch_id: str) -> dict[str, Any]:
|
||||
"""Run the full gate + bench sequence on a proposed patch and return
|
||||
the ledger row (gates, bench CIs, verdict). This is the only way the
|
||||
agent ever learns whether a change 'worked' — there is no other
|
||||
number to optimize."""
|
||||
raise HarnessUnavailable("verdict IPC not wired (Phase 1)")
|
||||
@@ -0,0 +1,18 @@
|
||||
[project]
|
||||
name = "hotpath-agent"
|
||||
version = "0.1.0"
|
||||
description = "hotpath optimization agent: the untrusted proposer in the profile→patch→verdict loop"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"claude-agent-sdk>=0.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = ["pytest>=8"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["hotpath_agent"]
|
||||
@@ -0,0 +1,20 @@
|
||||
# Accept thresholds for bench-runner (SPEC §3.1).
|
||||
# This file is trust-layer: read-only to the agent (SPEC §10).
|
||||
|
||||
# CI lower bound of the speedup ratio must clear 1 + threshold.
|
||||
threshold = 0.02
|
||||
|
||||
# Measured runs per side (SPEC minimum: 30). Interleaved ABAB.
|
||||
runs_per_side = 30
|
||||
|
||||
# Warm-up runs per side, discarded.
|
||||
warmup_runs = 3
|
||||
|
||||
# Bootstrap resamples for the ratio CI.
|
||||
bootstrap_iters = 10000
|
||||
|
||||
# Confidence level for the ratio CI.
|
||||
confidence = 0.95
|
||||
|
||||
# Fixed PRNG seed so calibration runs are reproducible.
|
||||
bootstrap_seed = 1887343589
|
||||
@@ -0,0 +1,12 @@
|
||||
# ROI pricing inputs (SPEC §3.6, §9). Defaults are public-cloud list
|
||||
# prices; replace with the customer's own numbers per engagement.
|
||||
# This file is trust-layer: read-only to the agent (SPEC §10).
|
||||
|
||||
# On-demand vCPU rate, dollars per core-hour.
|
||||
dollars_per_core_hour = 0.04
|
||||
|
||||
# Hours per year the fleet actually runs the workload (8760 = 24/7).
|
||||
hours_per_year = 8760.0
|
||||
|
||||
# Printed on the report so the number survives hostile review.
|
||||
rate_source = "example: AWS m7i on-demand us-east-1, per-vCPU approximation — replace per engagement"
|
||||
@@ -0,0 +1,15 @@
|
||||
# corpora/
|
||||
|
||||
Golden-replay input corpora and expected outputs, one directory per
|
||||
target. **Trust layer: read-only to the agent** (SPEC §10).
|
||||
|
||||
Every corpus directory carries a `MANIFEST.sha256` (`sha256sum` format:
|
||||
`<digest> <relative-path>` per line). `diff-test` verifies the manifest
|
||||
before every run and refuses to run on any mismatch — a tampered corpus
|
||||
is a stop-the-line event.
|
||||
|
||||
Regenerate a manifest only as a deliberate human action:
|
||||
|
||||
```sh
|
||||
cd corpora/<target> && find . -type f ! -name MANIFEST.sha256 -exec sha256sum {} + > MANIFEST.sha256
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
[package]
|
||||
name = "bench-runner"
|
||||
description = "Interleaved A/B benchmark harness with bootstrap CIs, A/A calibration, and environment fingerprinting"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
ledger = { path = "../ledger" }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
@@ -0,0 +1,48 @@
|
||||
//! Accept-threshold configuration, loaded from `config/accept.toml`.
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AcceptConfig {
|
||||
/// CI lower bound must clear 1 + threshold (default 2%).
|
||||
pub threshold: f64,
|
||||
/// Measured runs per side (SPEC minimum: 30).
|
||||
pub runs_per_side: usize,
|
||||
/// Unmeasured warm-up runs per side, discarded.
|
||||
pub warmup_runs: usize,
|
||||
/// Bootstrap resample count.
|
||||
pub bootstrap_iters: usize,
|
||||
/// Confidence level for the ratio CI, e.g. 0.95.
|
||||
pub confidence: f64,
|
||||
/// PRNG seed for the bootstrap (reproducible calibration).
|
||||
pub bootstrap_seed: u64,
|
||||
}
|
||||
|
||||
impl Default for AcceptConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
threshold: 0.02,
|
||||
runs_per_side: 30,
|
||||
warmup_runs: 3,
|
||||
bootstrap_iters: 10_000,
|
||||
confidence: 0.95,
|
||||
bootstrap_seed: 0x707E_17E5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AcceptConfig {
|
||||
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||
let raw = std::fs::read_to_string(path)?;
|
||||
let cfg: Self = toml::from_str(&raw)?;
|
||||
anyhow::ensure!(cfg.runs_per_side >= 2, "runs_per_side must be >= 2");
|
||||
anyhow::ensure!(cfg.threshold > 0.0, "threshold must be positive");
|
||||
anyhow::ensure!(
|
||||
(0.0..1.0).contains(&cfg.confidence),
|
||||
"confidence must be in (0, 1)"
|
||||
);
|
||||
Ok(cfg)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Command execution and timing. Whole-program mode (hyperfine-style):
|
||||
//! spawn the command, measure wall time, discard warm-ups per the
|
||||
//! schedule. `perf stat` counters and RAPL energy are Phase 1 follow-ups
|
||||
//! (tracked in README roadmap); wall time alone is enough to bring up
|
||||
//! A/A calibration.
|
||||
|
||||
use crate::schedule::{interleaved, Side};
|
||||
use anyhow::Context;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MeasuredSamples {
|
||||
pub baseline_s: Vec<f64>,
|
||||
pub candidate_s: Vec<f64>,
|
||||
}
|
||||
|
||||
fn time_once(shell_cmd: &str) -> anyhow::Result<f64> {
|
||||
let start = Instant::now();
|
||||
let status = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(shell_cmd)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.with_context(|| format!("failed to spawn: {shell_cmd}"))?;
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
anyhow::ensure!(status.success(), "command exited nonzero: {shell_cmd}");
|
||||
Ok(elapsed)
|
||||
}
|
||||
|
||||
/// Run baseline and candidate commands on the interleaved schedule and
|
||||
/// return measured wall times per side. For A/A mode, pass the same
|
||||
/// command as both sides.
|
||||
pub fn run_interleaved(
|
||||
baseline_cmd: &str,
|
||||
candidate_cmd: &str,
|
||||
runs_per_side: usize,
|
||||
warmup: usize,
|
||||
) -> anyhow::Result<MeasuredSamples> {
|
||||
let mut out = MeasuredSamples {
|
||||
baseline_s: Vec::with_capacity(runs_per_side),
|
||||
candidate_s: Vec::with_capacity(runs_per_side),
|
||||
};
|
||||
for (side, measured) in interleaved(runs_per_side, warmup) {
|
||||
let cmd = match side {
|
||||
Side::Baseline => baseline_cmd,
|
||||
Side::Candidate => candidate_cmd,
|
||||
};
|
||||
let elapsed = time_once(cmd)?;
|
||||
if measured {
|
||||
match side {
|
||||
Side::Baseline => out.baseline_s.push(elapsed),
|
||||
Side::Candidate => out.candidate_s.push(elapsed),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
//! Environment fingerprint (SPEC §3.1, §7): every reported number carries
|
||||
//! the machine state it was measured under. Collection is best-effort —
|
||||
//! a field we cannot read is recorded as "unknown", never omitted.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::fs;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct EnvFingerprint {
|
||||
pub kernel: String,
|
||||
pub cpu_model: String,
|
||||
pub governor: String,
|
||||
pub smt: String,
|
||||
pub turbo: String,
|
||||
/// ASLR handling mode actually in effect for this session (SPEC §7:
|
||||
/// randomize-and-aggregate vs. fixed; document, don't assume).
|
||||
pub aslr: String,
|
||||
pub cpu_pinning: String,
|
||||
}
|
||||
|
||||
fn read_trimmed(path: &str) -> String {
|
||||
fs::read_to_string(path)
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_else(|_| "unknown".to_string())
|
||||
}
|
||||
|
||||
impl EnvFingerprint {
|
||||
pub fn collect(pinning: &str, aslr_mode: &str) -> Self {
|
||||
let cpu_model = fs::read_to_string("/proc/cpuinfo")
|
||||
.ok()
|
||||
.and_then(|s| {
|
||||
s.lines()
|
||||
.find(|l| l.starts_with("model name"))
|
||||
.and_then(|l| l.split(':').nth(1))
|
||||
.map(|v| v.trim().to_string())
|
||||
})
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
Self {
|
||||
kernel: read_trimmed("/proc/sys/kernel/osrelease"),
|
||||
cpu_model,
|
||||
governor: read_trimmed(
|
||||
"/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
|
||||
),
|
||||
smt: read_trimmed("/sys/devices/system/cpu/smt/active"),
|
||||
turbo: read_trimmed("/sys/devices/system/cpu/intel_pstate/no_turbo"),
|
||||
aslr: format!(
|
||||
"mode={aslr_mode} randomize_va_space={}",
|
||||
read_trimmed("/proc/sys/kernel/randomize_va_space")
|
||||
),
|
||||
cpu_pinning: pinning.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
//! bench-runner core (SPEC §3.1, §7).
|
||||
//!
|
||||
//! The product is this crate: interleaved A/B measurement, bootstrap CIs
|
||||
//! over the ratio of medians, threshold-gated accept decisions, and A/A
|
||||
//! self-calibration. Pure statistics live here and are unit-tested;
|
||||
//! process execution and `perf stat` integration live in `exec`.
|
||||
|
||||
pub mod config;
|
||||
pub mod exec;
|
||||
pub mod fingerprint;
|
||||
pub mod schedule;
|
||||
pub mod stats;
|
||||
|
||||
use ledger::Verdict;
|
||||
|
||||
/// Decide a verdict from a speedup CI and the accept threshold.
|
||||
///
|
||||
/// Speedup is baseline/candidate (>1 means the candidate is faster).
|
||||
/// Accepted only if the CI *lower bound* clears `1 + threshold` — the
|
||||
/// point estimate is never enough (SPEC §3.1).
|
||||
pub fn decide(speedup_ci: stats::RatioCi, threshold: f64) -> Verdict {
|
||||
if speedup_ci.lo >= 1.0 + threshold {
|
||||
Verdict::Accepted
|
||||
} else {
|
||||
Verdict::RejectedBench
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::stats::RatioCi;
|
||||
|
||||
#[test]
|
||||
fn accept_requires_ci_lower_bound_not_point_estimate() {
|
||||
// 10% median speedup but the CI dips below threshold: rejected.
|
||||
let noisy = RatioCi { median: 1.10, lo: 1.005, hi: 1.21 };
|
||||
assert_eq!(decide(noisy, 0.02), Verdict::RejectedBench);
|
||||
// 4% speedup with a tight CI: accepted.
|
||||
let tight = RatioCi { median: 1.04, lo: 1.025, hi: 1.055 };
|
||||
assert_eq!(decide(tight, 0.02), Verdict::Accepted);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
use anyhow::Result;
|
||||
use bench_runner::{config::AcceptConfig, decide, exec, fingerprint::EnvFingerprint, stats};
|
||||
use clap::{Parser, Subcommand};
|
||||
use ledger::Verdict;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "bench-runner", about = "Interleaved A/B benchmark harness (hotpath trust layer)")]
|
||||
struct Cli {
|
||||
/// Path to accept.toml; built-in defaults if omitted.
|
||||
#[arg(long)]
|
||||
config: Option<PathBuf>,
|
||||
#[command(subcommand)]
|
||||
command: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Interleaved A/B comparison of two shell commands.
|
||||
Compare {
|
||||
#[arg(long)]
|
||||
baseline: String,
|
||||
#[arg(long)]
|
||||
candidate: String,
|
||||
},
|
||||
/// A/A self-test: same command both sides; must yield a null verdict.
|
||||
Aa {
|
||||
#[arg(long)]
|
||||
cmd: String,
|
||||
},
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let cfg = match &cli.config {
|
||||
Some(p) => AcceptConfig::load(p)?,
|
||||
None => AcceptConfig::default(),
|
||||
};
|
||||
|
||||
let (baseline, candidate, aa_mode) = match &cli.command {
|
||||
Cmd::Compare { baseline, candidate } => (baseline.clone(), candidate.clone(), false),
|
||||
Cmd::Aa { cmd } => (cmd.clone(), cmd.clone(), true),
|
||||
};
|
||||
|
||||
let fp = EnvFingerprint::collect("none", "system-default");
|
||||
eprintln!("env fingerprint: {}", serde_json::to_string(&fp)?);
|
||||
eprintln!(
|
||||
"plan: {} measured + {} warm-up runs per side, interleaved",
|
||||
cfg.runs_per_side, cfg.warmup_runs
|
||||
);
|
||||
|
||||
let samples =
|
||||
exec::run_interleaved(&baseline, &candidate, cfg.runs_per_side, cfg.warmup_runs)?;
|
||||
let ci = stats::bootstrap_ratio_ci(
|
||||
&samples.baseline_s,
|
||||
&samples.candidate_s,
|
||||
cfg.bootstrap_iters,
|
||||
cfg.confidence,
|
||||
cfg.bootstrap_seed,
|
||||
);
|
||||
|
||||
println!(
|
||||
"speedup (baseline/candidate): median {:.4}, {:.0}% CI [{:.4}, {:.4}]",
|
||||
ci.median,
|
||||
cfg.confidence * 100.0,
|
||||
ci.lo,
|
||||
ci.hi
|
||||
);
|
||||
|
||||
if aa_mode {
|
||||
// Null verdict required: an "accept" here is a calibration failure.
|
||||
let verdict = decide(ci, cfg.threshold);
|
||||
if verdict == Verdict::Accepted {
|
||||
anyhow::bail!("A/A self-test FAILED: harness claims a speedup from identical binaries");
|
||||
}
|
||||
println!("A/A self-test passed (null verdict)");
|
||||
} else {
|
||||
println!("verdict: {}", decide(ci, cfg.threshold).as_str());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
//! Interleaved run scheduling (SPEC §3.1): baseline and candidate run
|
||||
//! ABABAB…, never sequential blocks, so thermal and background drift
|
||||
//! land on both sides equally.
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Side {
|
||||
Baseline,
|
||||
Candidate,
|
||||
}
|
||||
|
||||
/// ABAB… schedule with `runs_per_side` measured runs on each side,
|
||||
/// preceded by `warmup` unmeasured warm-up entries (alternating too, so
|
||||
/// both binaries reach steady state).
|
||||
pub fn interleaved(runs_per_side: usize, warmup: usize) -> Vec<(Side, bool)> {
|
||||
let mut plan = Vec::with_capacity(2 * (runs_per_side + warmup));
|
||||
for i in 0..(runs_per_side + warmup) {
|
||||
let measured = i >= warmup;
|
||||
plan.push((Side::Baseline, measured));
|
||||
plan.push((Side::Candidate, measured));
|
||||
}
|
||||
plan
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strictly_alternating_and_counts_match() {
|
||||
let plan = interleaved(30, 3);
|
||||
assert_eq!(plan.len(), 66);
|
||||
for pair in plan.chunks(2) {
|
||||
assert_eq!(pair[0].0, Side::Baseline);
|
||||
assert_eq!(pair[1].0, Side::Candidate);
|
||||
}
|
||||
let measured_baseline = plan
|
||||
.iter()
|
||||
.filter(|(s, m)| *s == Side::Baseline && *m)
|
||||
.count();
|
||||
assert_eq!(measured_baseline, 30);
|
||||
assert!(!plan[0].1 && !plan[5].1, "warm-ups are unmeasured");
|
||||
assert!(plan[6].1, "runs after warm-up are measured");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
//! Ratios with bootstrap CIs; never means without spread; never a single
|
||||
//! run, ever (SPEC §7).
|
||||
|
||||
/// Bootstrap confidence interval over the ratio of medians
|
||||
/// (baseline / candidate).
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct RatioCi {
|
||||
pub median: f64,
|
||||
pub lo: f64,
|
||||
pub hi: f64,
|
||||
}
|
||||
|
||||
pub fn median(xs: &[f64]) -> f64 {
|
||||
assert!(!xs.is_empty(), "median of empty sample");
|
||||
let mut v = xs.to_vec();
|
||||
v.sort_by(|a, b| a.partial_cmp(b).expect("NaN in sample"));
|
||||
let n = v.len();
|
||||
if n % 2 == 1 {
|
||||
v[n / 2]
|
||||
} else {
|
||||
(v[n / 2 - 1] + v[n / 2]) / 2.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministic xorshift64* PRNG. Seeded explicitly so calibration runs
|
||||
/// are reproducible; no wall-clock entropy anywhere in the stats path.
|
||||
struct XorShift64(u64);
|
||||
|
||||
impl XorShift64 {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self(seed.max(1))
|
||||
}
|
||||
fn next_u64(&mut self) -> u64 {
|
||||
let mut x = self.0;
|
||||
x ^= x >> 12;
|
||||
x ^= x << 25;
|
||||
x ^= x >> 27;
|
||||
self.0 = x;
|
||||
x.wrapping_mul(0x2545F4914F6CDD1D)
|
||||
}
|
||||
fn index(&mut self, n: usize) -> usize {
|
||||
(self.next_u64() % n as u64) as usize
|
||||
}
|
||||
}
|
||||
|
||||
/// Percentile-method bootstrap CI of median(baseline)/median(candidate).
|
||||
///
|
||||
/// `confidence` is e.g. 0.95. Both samples are resampled independently
|
||||
/// with replacement `iters` times.
|
||||
pub fn bootstrap_ratio_ci(
|
||||
baseline: &[f64],
|
||||
candidate: &[f64],
|
||||
iters: usize,
|
||||
confidence: f64,
|
||||
seed: u64,
|
||||
) -> RatioCi {
|
||||
assert!(baseline.len() >= 2 && candidate.len() >= 2, "need >=2 runs per side");
|
||||
assert!((0.0..1.0).contains(&confidence));
|
||||
let mut rng = XorShift64::new(seed);
|
||||
let mut ratios = Vec::with_capacity(iters);
|
||||
let mut b_resample = vec![0.0; baseline.len()];
|
||||
let mut c_resample = vec![0.0; candidate.len()];
|
||||
for _ in 0..iters {
|
||||
for slot in b_resample.iter_mut() {
|
||||
*slot = baseline[rng.index(baseline.len())];
|
||||
}
|
||||
for slot in c_resample.iter_mut() {
|
||||
*slot = candidate[rng.index(candidate.len())];
|
||||
}
|
||||
ratios.push(median(&b_resample) / median(&c_resample));
|
||||
}
|
||||
ratios.sort_by(|a, b| a.partial_cmp(b).expect("NaN ratio"));
|
||||
let alpha = (1.0 - confidence) / 2.0;
|
||||
let lo_idx = ((iters as f64) * alpha).floor() as usize;
|
||||
let hi_idx = (((iters as f64) * (1.0 - alpha)).ceil() as usize).min(iters - 1);
|
||||
RatioCi {
|
||||
median: median(baseline) / median(candidate),
|
||||
lo: ratios[lo_idx],
|
||||
hi: ratios[hi_idx],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn median_odd_and_even() {
|
||||
assert_eq!(median(&[3.0, 1.0, 2.0]), 2.0);
|
||||
assert_eq!(median(&[4.0, 1.0, 2.0, 3.0]), 2.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aa_ci_straddles_one() {
|
||||
// Same distribution both sides: the CI must contain 1.0.
|
||||
let sample: Vec<f64> = (0..40).map(|i| 1.0 + 0.01 * ((i * 7) % 11) as f64).collect();
|
||||
let ci = bootstrap_ratio_ci(&sample, &sample, 2000, 0.95, 42);
|
||||
assert!(ci.lo <= 1.0 && ci.hi >= 1.0, "A/A CI must straddle 1.0, got {ci:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_injected_regression() {
|
||||
// Candidate 5% slower with mild noise: speedup CI entirely < 1.
|
||||
let base: Vec<f64> = (0..40).map(|i| 1.0 + 0.002 * ((i * 3) % 7) as f64).collect();
|
||||
let cand: Vec<f64> = base.iter().map(|t| t * 1.05).collect();
|
||||
let ci = bootstrap_ratio_ci(&base, &cand, 2000, 0.95, 42);
|
||||
assert!(ci.hi < 1.0, "5% regression must be detected, got {ci:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_for_fixed_seed() {
|
||||
let a = vec![1.0, 1.1, 1.05, 0.98, 1.02, 1.07];
|
||||
let b = vec![0.9, 0.95, 0.93, 0.91, 0.97, 0.92];
|
||||
let x = bootstrap_ratio_ci(&a, &b, 500, 0.95, 7);
|
||||
let y = bootstrap_ratio_ci(&a, &b, 500, 0.95, 7);
|
||||
assert_eq!(x, y);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "diff-test"
|
||||
description = "Layered equivalence gates: upstream tests, golden replay, differential fuzzing, sanitizers; hash-pinned corpora"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
serde.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
toml.workspace = true
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Equivalence gates (SPEC §3.2) and corpus hash-pinning.
|
||||
//!
|
||||
//! The gate *sequencing* (upstream tests → golden replay → differential
|
||||
//! fuzz → sanitizers) is orchestrated per-target; this crate owns the
|
||||
//! pieces that must be trustworthy regardless of target: the pin
|
||||
//! verifier that refuses to run on a tampered corpus, and the
|
||||
//! per-target equivalence policy parser (byte-identical unless a target
|
||||
//! declares an explicit FP tolerance).
|
||||
|
||||
pub mod pin;
|
||||
pub mod policy;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Outcome of one gate layer. `Skipped` is only legal where the SPEC
|
||||
/// allows it (e.g. TSan when no concurrency primitives changed) and the
|
||||
/// reason is always recorded.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum GateOutcome {
|
||||
Passed,
|
||||
Failed { detail: String },
|
||||
Skipped { reason: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum GateLayer {
|
||||
UpstreamTests,
|
||||
GoldenReplay,
|
||||
DifferentialFuzz,
|
||||
Sanitizers,
|
||||
}
|
||||
|
||||
/// All layers must pass; any failure is an auto-reject (SPEC §8).
|
||||
pub fn all_passed(results: &[(GateLayer, GateOutcome)]) -> bool {
|
||||
results.iter().all(|(_, o)| {
|
||||
matches!(o, GateOutcome::Passed | GateOutcome::Skipped { .. })
|
||||
}) && results
|
||||
.iter()
|
||||
.any(|(_, o)| matches!(o, GateOutcome::Passed))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn any_failure_rejects() {
|
||||
let results = vec![
|
||||
(GateLayer::UpstreamTests, GateOutcome::Passed),
|
||||
(
|
||||
GateLayer::GoldenReplay,
|
||||
GateOutcome::Failed { detail: "stdout diverged".into() },
|
||||
),
|
||||
];
|
||||
assert!(!all_passed(&results));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_skipped_is_not_a_pass() {
|
||||
let results = vec![(
|
||||
GateLayer::Sanitizers,
|
||||
GateOutcome::Skipped { reason: "n/a".into() },
|
||||
)];
|
||||
assert!(!all_passed(&results));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
//! Corpus hash-pinning (SPEC §3.2, §10): corpora live outside
|
||||
//! agent-writable paths and are pinned by a SHA-256 manifest. diff-test
|
||||
//! refuses to run if any hash mismatches — a tampered corpus is a
|
||||
//! stop-the-line event, not a warning.
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum PinError {
|
||||
#[error("io error reading {path}: {source}")]
|
||||
Io {
|
||||
path: String,
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("malformed manifest line {line}: {content:?}")]
|
||||
MalformedManifest { line: usize, content: String },
|
||||
#[error("HASH MISMATCH for {path}: manifest {expected}, actual {actual} — refusing to run")]
|
||||
Mismatch {
|
||||
path: String,
|
||||
expected: String,
|
||||
actual: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub fn sha256_hex(path: &Path) -> Result<String, PinError> {
|
||||
let bytes = std::fs::read(path).map_err(|source| PinError::Io {
|
||||
path: path.display().to_string(),
|
||||
source,
|
||||
})?;
|
||||
Ok(format!("{:x}", Sha256::digest(&bytes)))
|
||||
}
|
||||
|
||||
/// Verify every entry of a `sha256sum`-format manifest
|
||||
/// (`<hex-digest> <relative-path>` per line, paths relative to `root`).
|
||||
/// Returns the number of files verified; errors on the first mismatch.
|
||||
pub fn verify_manifest(manifest: &Path, root: &Path) -> Result<usize, PinError> {
|
||||
let raw = std::fs::read_to_string(manifest).map_err(|source| PinError::Io {
|
||||
path: manifest.display().to_string(),
|
||||
source,
|
||||
})?;
|
||||
let mut verified = 0;
|
||||
for (i, line) in raw.lines().enumerate() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let (expected, rel) = line
|
||||
.split_once(" ")
|
||||
.ok_or_else(|| PinError::MalformedManifest {
|
||||
line: i + 1,
|
||||
content: line.to_string(),
|
||||
})?;
|
||||
let path = root.join(rel);
|
||||
let actual = sha256_hex(&path)?;
|
||||
if !actual.eq_ignore_ascii_case(expected) {
|
||||
return Err(PinError::Mismatch {
|
||||
path: path.display().to_string(),
|
||||
expected: expected.to_string(),
|
||||
actual,
|
||||
});
|
||||
}
|
||||
verified += 1;
|
||||
}
|
||||
Ok(verified)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn verifies_and_refuses() {
|
||||
let dir = std::env::temp_dir().join("hotpath-pin-test");
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
let corpus = dir.join("input.bin");
|
||||
fs::write(&corpus, b"golden input").unwrap();
|
||||
let digest = sha256_hex(&corpus).unwrap();
|
||||
|
||||
let manifest = dir.join("MANIFEST.sha256");
|
||||
fs::write(&manifest, format!("{digest} input.bin\n")).unwrap();
|
||||
assert_eq!(verify_manifest(&manifest, &dir).unwrap(), 1);
|
||||
|
||||
// Tamper: verification must refuse.
|
||||
fs::write(&corpus, b"tampered input").unwrap();
|
||||
let err = verify_manifest(&manifest, &dir).unwrap_err();
|
||||
assert!(matches!(err, PinError::Mismatch { .. }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
//! Per-target equivalence policy (`targets/<name>/equivalence.toml`).
|
||||
//! Absent a policy file, byte-identical is required (SPEC §3.2).
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)]
|
||||
pub enum EquivalencePolicy {
|
||||
/// Default: stdout, output files, and exit codes must match exactly.
|
||||
ByteIdentical,
|
||||
/// FP-producing targets only: numeric fields may differ within the
|
||||
/// declared tolerance; everything else stays byte-identical. Using
|
||||
/// this tier at all routes the attempt to needs-human-review when
|
||||
/// FP *flags* changed (SPEC §8) — tolerance covers output compare,
|
||||
/// not compiler-flag laxity.
|
||||
FpTolerance {
|
||||
/// Maximum absolute difference per numeric value.
|
||||
abs: f64,
|
||||
/// Maximum relative difference per numeric value.
|
||||
rel: f64,
|
||||
},
|
||||
}
|
||||
|
||||
impl EquivalencePolicy {
|
||||
/// Load the target's policy, defaulting to byte-identical when no
|
||||
/// equivalence.toml exists.
|
||||
pub fn load(target_dir: &Path) -> anyhow::Result<Self> {
|
||||
let path = target_dir.join("equivalence.toml");
|
||||
if !path.exists() {
|
||||
return Ok(Self::ByteIdentical);
|
||||
}
|
||||
let raw = std::fs::read_to_string(&path)?;
|
||||
let policy: Self = toml::from_str(&raw)?;
|
||||
if let Self::FpTolerance { abs, rel } = &policy {
|
||||
anyhow::ensure!(
|
||||
*abs >= 0.0 && *rel >= 0.0,
|
||||
"tolerances must be non-negative"
|
||||
);
|
||||
}
|
||||
Ok(policy)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn absent_policy_means_byte_identical() {
|
||||
let dir = std::env::temp_dir().join("hotpath-policy-none");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
assert_eq!(
|
||||
EquivalencePolicy::load(&dir).unwrap(),
|
||||
EquivalencePolicy::ByteIdentical
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_fp_tolerance() {
|
||||
let dir = std::env::temp_dir().join("hotpath-policy-fp");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(
|
||||
dir.join("equivalence.toml"),
|
||||
"mode = \"fp-tolerance\"\nabs = 1e-9\nrel = 1e-6\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
EquivalencePolicy::load(&dir).unwrap(),
|
||||
EquivalencePolicy::FpTolerance { abs: 1e-9, rel: 1e-6 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "ledger"
|
||||
description = "Append-only SQLite attempt ledger: audit trail, anti-double-attempt memory, future training set"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Append-only attempt ledger (SPEC §3.3).
|
||||
//!
|
||||
//! One row per optimization attempt. Nothing is ever deleted or updated —
|
||||
//! enforced with SQLite triggers, not convention. The ledger is the audit
|
||||
//! trail, the anti-double-attempt memory, and the future training set.
|
||||
|
||||
use rusqlite::Connection;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LedgerError {
|
||||
#[error("sqlite error: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
#[error("serialization error: {0}")]
|
||||
Serde(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum Verdict {
|
||||
Accepted,
|
||||
RejectedGate,
|
||||
RejectedBench,
|
||||
NeedsHumanReview,
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Verdict::Accepted => "accepted",
|
||||
Verdict::RejectedGate => "rejected-gate",
|
||||
Verdict::RejectedBench => "rejected-bench",
|
||||
Verdict::NeedsHumanReview => "needs-human-review",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gate outcomes for one attempt (SPEC §3.2).
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct GateResults {
|
||||
pub upstream_tests: bool,
|
||||
pub golden_replay: bool,
|
||||
pub fuzz_iters: u64,
|
||||
pub fuzz_divergence: bool,
|
||||
pub sanitizers_clean: bool,
|
||||
}
|
||||
|
||||
/// Benchmark evidence for one attempt. All times in seconds; CIs are
|
||||
/// bootstrap 95% intervals over the ratio baseline/candidate.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct BenchEvidence {
|
||||
pub baseline_median: f64,
|
||||
pub baseline_ci: (f64, f64),
|
||||
pub candidate_median: f64,
|
||||
pub candidate_ci: (f64, f64),
|
||||
pub speedup_median: f64,
|
||||
pub speedup_ci: (f64, f64),
|
||||
pub env_fingerprint: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Attempt {
|
||||
pub run_id: String,
|
||||
/// RFC 3339, supplied by the harness (the ledger records, it does not clock).
|
||||
pub timestamp: String,
|
||||
pub target: String,
|
||||
pub target_commit: String,
|
||||
pub phase: u8,
|
||||
/// Symbol plus its share of the profile, e.g. "png_read_filter_row (31.4%)".
|
||||
pub hotspot: String,
|
||||
/// Playbook class 1–7 (SPEC §4).
|
||||
pub playbook_class: u8,
|
||||
pub hypothesis: String,
|
||||
/// Unified diff as proposed by the agent, pre-application.
|
||||
pub patch: String,
|
||||
pub gates: GateResults,
|
||||
pub bench: Option<BenchEvidence>,
|
||||
pub verdict: Verdict,
|
||||
pub tokens_spent: u64,
|
||||
pub wall_time_s: f64,
|
||||
}
|
||||
|
||||
pub struct Ledger {
|
||||
conn: Connection,
|
||||
}
|
||||
|
||||
const SCHEMA: &str = r#"
|
||||
CREATE TABLE IF NOT EXISTS attempts (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
timestamp TEXT NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
target_commit TEXT NOT NULL,
|
||||
phase INTEGER NOT NULL,
|
||||
hotspot TEXT NOT NULL,
|
||||
playbook_class INTEGER NOT NULL,
|
||||
hypothesis TEXT NOT NULL,
|
||||
patch TEXT NOT NULL,
|
||||
gates TEXT NOT NULL, -- JSON GateResults
|
||||
bench TEXT, -- JSON BenchEvidence, NULL if gates failed first
|
||||
verdict TEXT NOT NULL CHECK (verdict IN
|
||||
('accepted','rejected-gate','rejected-bench','needs-human-review')),
|
||||
tokens_spent INTEGER NOT NULL,
|
||||
wall_time_s REAL NOT NULL
|
||||
);
|
||||
|
||||
-- Append-only, structurally: mutation raises instead of succeeding.
|
||||
CREATE TRIGGER IF NOT EXISTS attempts_no_update
|
||||
BEFORE UPDATE ON attempts
|
||||
BEGIN SELECT RAISE(ABORT, 'ledger is append-only'); END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS attempts_no_delete
|
||||
BEFORE DELETE ON attempts
|
||||
BEGIN SELECT RAISE(ABORT, 'ledger is append-only'); END;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attempts_by_target ON attempts (target, playbook_class);
|
||||
"#;
|
||||
|
||||
impl Ledger {
|
||||
pub fn open(path: &Path) -> Result<Self, LedgerError> {
|
||||
let conn = Connection::open(path)?;
|
||||
conn.execute_batch(SCHEMA)?;
|
||||
Ok(Self { conn })
|
||||
}
|
||||
|
||||
pub fn open_in_memory() -> Result<Self, LedgerError> {
|
||||
let conn = Connection::open_in_memory()?;
|
||||
conn.execute_batch(SCHEMA)?;
|
||||
Ok(Self { conn })
|
||||
}
|
||||
|
||||
pub fn record(&self, a: &Attempt) -> Result<(), LedgerError> {
|
||||
let bench_json = a.bench.as_ref().map(serde_json::to_string).transpose()?;
|
||||
self.conn.execute(
|
||||
"INSERT INTO attempts (run_id, timestamp, target, target_commit, phase, hotspot,
|
||||
playbook_class, hypothesis, patch, gates, bench, verdict, tokens_spent, wall_time_s)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)",
|
||||
rusqlite::params![
|
||||
a.run_id,
|
||||
a.timestamp,
|
||||
a.target,
|
||||
a.target_commit,
|
||||
a.phase,
|
||||
a.hotspot,
|
||||
a.playbook_class,
|
||||
a.hypothesis,
|
||||
a.patch,
|
||||
serde_json::to_string(&a.gates)?,
|
||||
bench_json,
|
||||
a.verdict.as_str(),
|
||||
a.tokens_spent,
|
||||
a.wall_time_s,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Playbook classes already attempted against a target — the agent's
|
||||
/// anti-double-attempt memory (SPEC §3.5 prompting spine, rule 2).
|
||||
pub fn attempted_classes(&self, target: &str) -> Result<Vec<u8>, LedgerError> {
|
||||
let mut stmt = self.conn.prepare(
|
||||
"SELECT DISTINCT playbook_class FROM attempts WHERE target = ?1 ORDER BY playbook_class",
|
||||
)?;
|
||||
let rows = stmt.query_map([target], |r| r.get::<_, u8>(0))?;
|
||||
Ok(rows.collect::<Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
pub fn count(&self) -> Result<u64, LedgerError> {
|
||||
Ok(self
|
||||
.conn
|
||||
.query_row("SELECT COUNT(*) FROM attempts", [], |r| r.get(0))?)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_attempt(run_id: &str) -> Attempt {
|
||||
Attempt {
|
||||
run_id: run_id.into(),
|
||||
timestamp: "2026-07-06T00:00:00Z".into(),
|
||||
target: "example".into(),
|
||||
target_commit: "deadbeef".into(),
|
||||
phase: 1,
|
||||
hotspot: "hot_fn (42.0%)".into(),
|
||||
playbook_class: 2,
|
||||
hypothesis: "mimalloc reduces allocator contention".into(),
|
||||
patch: "--- a/Cargo.toml\n+++ b/Cargo.toml\n".into(),
|
||||
gates: GateResults::default(),
|
||||
bench: None,
|
||||
verdict: Verdict::RejectedGate,
|
||||
tokens_spent: 1000,
|
||||
wall_time_s: 12.5,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_and_query() {
|
||||
let ledger = Ledger::open_in_memory().unwrap();
|
||||
ledger.record(&sample_attempt("r1")).unwrap();
|
||||
assert_eq!(ledger.count().unwrap(), 1);
|
||||
assert_eq!(ledger.attempted_classes("example").unwrap(), vec![2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_only_is_enforced() {
|
||||
let ledger = Ledger::open_in_memory().unwrap();
|
||||
ledger.record(&sample_attempt("r1")).unwrap();
|
||||
let upd = ledger
|
||||
.conn
|
||||
.execute("UPDATE attempts SET verdict = 'accepted'", []);
|
||||
assert!(upd.is_err(), "UPDATE must be rejected by trigger");
|
||||
let del = ledger.conn.execute("DELETE FROM attempts", []);
|
||||
assert!(del.is_err(), "DELETE must be rejected by trigger");
|
||||
assert_eq!(ledger.count().unwrap(), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "report"
|
||||
description = "ROI reports from ledger rows: throughput→cores→dollars with CIs, workload statement, and env fingerprint on every figure"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
ledger = { path = "../ledger" }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
toml.workspace = true
|
||||
@@ -0,0 +1,74 @@
|
||||
//! ROI methodology (SPEC §3.6, §9): what did the stopwatch say, and what
|
||||
//! does that cost or buy? Every figure carries its CI; the caveats print
|
||||
//! on the report, not in an appendix.
|
||||
|
||||
pub mod pricing;
|
||||
|
||||
/// Cores returned to the fleet by a throughput speedup:
|
||||
/// `fleet_cores × (1 − 1/speedup)`.
|
||||
pub fn cores_saved(fleet_cores: f64, speedup: f64) -> f64 {
|
||||
assert!(speedup > 0.0, "speedup must be positive");
|
||||
fleet_cores * (1.0 - 1.0 / speedup)
|
||||
}
|
||||
|
||||
/// Annual dollars from saved cores at a $/core-hour rate.
|
||||
pub fn dollars_per_year(cores: f64, dollars_per_core_hour: f64, hours_per_year: f64) -> f64 {
|
||||
cores * dollars_per_core_hour * hours_per_year
|
||||
}
|
||||
|
||||
/// ROI range induced by a speedup CI — the report quotes the interval,
|
||||
/// never just the point estimate.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct RoiRange {
|
||||
pub cores_lo: f64,
|
||||
pub cores_median: f64,
|
||||
pub cores_hi: f64,
|
||||
pub dollars_lo: f64,
|
||||
pub dollars_median: f64,
|
||||
pub dollars_hi: f64,
|
||||
}
|
||||
|
||||
pub fn roi_from_speedup_ci(
|
||||
fleet_cores: f64,
|
||||
speedup_median: f64,
|
||||
speedup_ci: (f64, f64),
|
||||
dollars_per_core_hour: f64,
|
||||
hours_per_year: f64,
|
||||
) -> RoiRange {
|
||||
let (lo, hi) = speedup_ci;
|
||||
let (c_lo, c_med, c_hi) = (
|
||||
cores_saved(fleet_cores, lo),
|
||||
cores_saved(fleet_cores, speedup_median),
|
||||
cores_saved(fleet_cores, hi),
|
||||
);
|
||||
RoiRange {
|
||||
cores_lo: c_lo,
|
||||
cores_median: c_med,
|
||||
cores_hi: c_hi,
|
||||
dollars_lo: dollars_per_year(c_lo, dollars_per_core_hour, hours_per_year),
|
||||
dollars_median: dollars_per_year(c_med, dollars_per_core_hour, hours_per_year),
|
||||
dollars_hi: dollars_per_year(c_hi, dollars_per_core_hour, hours_per_year),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cores_saved_formula() {
|
||||
// 2x speedup halves the fleet: 100 cores → 50 saved.
|
||||
assert!((cores_saved(100.0, 2.0) - 50.0).abs() < 1e-12);
|
||||
// 1.25x on 1000 cores: 1000 × (1 − 0.8) = 200.
|
||||
assert!((cores_saved(1000.0, 1.25) - 200.0).abs() < 1e-9);
|
||||
// No speedup, no savings.
|
||||
assert_eq!(cores_saved(100.0, 1.0), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roi_range_is_ordered() {
|
||||
let r = roi_from_speedup_ci(1000.0, 1.10, (1.05, 1.16), 0.04, 8760.0);
|
||||
assert!(r.cores_lo < r.cores_median && r.cores_median < r.cores_hi);
|
||||
assert!(r.dollars_lo < r.dollars_median && r.dollars_median < r.dollars_hi);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
//! Pricing inputs (`config/pricing.toml`). Rates are engagement inputs,
|
||||
//! not constants — public cloud list prices by default, customer's own
|
||||
//! numbers when supplied.
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Pricing {
|
||||
/// Dollars per core-hour (e.g. public-cloud on-demand vCPU rate).
|
||||
pub dollars_per_core_hour: f64,
|
||||
/// Fleet hours per year the workload actually runs (8760 for 24/7).
|
||||
pub hours_per_year: f64,
|
||||
/// Where the rate came from — printed on the report.
|
||||
pub rate_source: String,
|
||||
}
|
||||
|
||||
impl Pricing {
|
||||
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||
let raw = std::fs::read_to_string(path)?;
|
||||
let p: Self = toml::from_str(&raw)?;
|
||||
anyhow::ensure!(p.dollars_per_core_hour > 0.0, "rate must be positive");
|
||||
anyhow::ensure!(
|
||||
p.hours_per_year > 0.0 && p.hours_per_year <= 8784.0,
|
||||
"hours_per_year must be in (0, 8784]"
|
||||
);
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
# hotpath task runner. `just --list` for a summary.
|
||||
|
||||
# Build the trust-layer workspace
|
||||
build:
|
||||
cargo build --workspace
|
||||
|
||||
# Run all trust-layer unit tests
|
||||
test:
|
||||
cargo test --workspace
|
||||
|
||||
# Lint: warnings are errors in the trust layer
|
||||
lint:
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
cargo fmt --all --check
|
||||
|
||||
# A/A self-test of the bench harness against a sample command.
|
||||
# Must print a null verdict; an "accept" is a calibration failure.
|
||||
aa cmd="sleep 0.05":
|
||||
cargo run -p bench-runner -- --config config/accept.toml aa --cmd "{{cmd}}"
|
||||
|
||||
# Interleaved A/B comparison of two shell commands
|
||||
compare baseline candidate:
|
||||
cargo run -p bench-runner -- --config config/accept.toml compare \
|
||||
--baseline "{{baseline}}" --candidate "{{candidate}}"
|
||||
|
||||
# Verify a target corpus against its pinned manifest
|
||||
# (wired through diff-test CLI once it exists; manual check for now)
|
||||
pin-check target:
|
||||
cd corpora/{{target}} && sha256sum -c MANIFEST.sha256
|
||||
@@ -0,0 +1,41 @@
|
||||
# Class 1 — Build configuration
|
||||
|
||||
Zero source changes. Always first: a large fraction of real-world CPU is
|
||||
lost to shipping defaults.
|
||||
|
||||
## Preconditions
|
||||
- Any CPU-bound target not already using an optimized release profile.
|
||||
- Profile signature: flat-ish flamegraph with time spread across many
|
||||
small frames (inlining starved); heavy call overhead into tiny leaf
|
||||
functions; or a build that visibly lacks `-O2`/`--release`.
|
||||
- Check before anything else: `opt-level`, `lto`, `codegen-units`,
|
||||
`panic`, `target-cpu` (Rust); `-O`, `-march`, `-flto` (C/C++).
|
||||
|
||||
## Procedure (cheapest first)
|
||||
1. Confirm release profile: `opt-level=3`, `panic=abort` where the target
|
||||
permits, `codegen-units=1`.
|
||||
2. `target-cpu=native` / `-march=native` — only when the deployment
|
||||
fleet's ISA baseline is known; record the chosen baseline in the
|
||||
hypothesis.
|
||||
3. LTO: thin, then fat. Watch build-time cost; fat LTO on huge targets
|
||||
can be a net loss for the engagement.
|
||||
4. PGO: instrument with the golden-replay corpus as the training run,
|
||||
rebuild with the profile. The corpus is representative by
|
||||
construction.
|
||||
5. BOLT (post-link layout) where the binary and toolchain permit.
|
||||
|
||||
## Verification notes
|
||||
- Flags must not include any `-ffast-math`-class option — that's an
|
||||
equivalence change, auto-routed to `needs-human-review` (SPEC §8).
|
||||
- Byte-identical golden replay still applies: build flags occasionally
|
||||
change FP contraction on some ISAs (`-march` enabling FMA); if golden
|
||||
replay diverges on numeric output, stop — this class is not
|
||||
equivalence-preserving for that target without a tolerance policy.
|
||||
|
||||
## Known failure modes
|
||||
- `target-cpu=native` on the bench machine but not the fleet: a real win
|
||||
that the customer cannot deploy. Pin the ISA baseline to the fleet.
|
||||
- PGO trained on a non-representative workload overfits the benchmark —
|
||||
hold out workloads (SPEC §11).
|
||||
- Layout changes from LTO/BOLT interact with layout bias (§7): measured
|
||||
wins here *especially* need the randomize-and-aggregate protocol.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Class 2 — Allocator swap
|
||||
|
||||
Link-line only. With class 1, forms the "your 30% was a linker flag"
|
||||
tier that is always exhausted before touching source.
|
||||
|
||||
## Preconditions
|
||||
- Profile signature: `malloc`/`free`/`operator new` or allocator-internal
|
||||
symbols (`_int_malloc`, `tcache_*`, page-fault handling) ≥5% of
|
||||
exclusive samples; or high syscall time in `mmap`/`brk`; or
|
||||
multithreaded target with allocator lock contention visible.
|
||||
|
||||
## Procedure
|
||||
1. Rust: add `mimalloc` (then `jemallocator`) as `#[global_allocator]` —
|
||||
two-line diff, still effectively link-level.
|
||||
2. C/C++: `LD_PRELOAD` trial first (free experiment, no rebuild), then
|
||||
proper linking of the winner.
|
||||
3. Bench each candidate allocator separately; do not stack with other
|
||||
changes.
|
||||
|
||||
## Verification notes
|
||||
- Behavior-preserving by contract, so gates should pass trivially — but
|
||||
run them all anyway: allocator swaps change address-space layout and
|
||||
can unmask latent UB (use-after-free that "worked" under glibc). A
|
||||
sanitizer failure here is a pre-existing target bug: record it,
|
||||
route to `needs-human-review`, do not ship.
|
||||
- Max RSS is captured per run: an allocator that wins time but doubles
|
||||
memory is a trade-off the report must state, not hide.
|
||||
|
||||
## Known failure modes
|
||||
- Wins on the bench box's core count that vanish (or invert) at the
|
||||
fleet's concurrency level.
|
||||
- `LD_PRELOAD` trial passing but static-link integration subtly failing
|
||||
(symbol interposition differences) — re-run full gates on the final
|
||||
link, not the trial.
|
||||
- Fragmentation pathologies on long-running services that a short bench
|
||||
window never sees; service-mode targets need soak evidence (Phase 4).
|
||||
@@ -0,0 +1,36 @@
|
||||
# Class 3 — Allocation churn
|
||||
|
||||
First source-touching class: reduce the *number* of allocations, not the
|
||||
allocator servicing them.
|
||||
|
||||
## Preconditions
|
||||
- Class 2 helped but allocator symbols still hot, or allocation shows up
|
||||
as cache misses from pointer-chasing freshly allocated objects.
|
||||
- Profile signature: allocation inside hot loops; `Vec`/`String`
|
||||
reallocation churn; many short-lived objects of the same type.
|
||||
|
||||
## Procedure (cheapest first)
|
||||
1. Hoist allocations out of loops; reuse buffers via `clear()` instead
|
||||
of re-creation.
|
||||
2. `Vec::with_capacity` / `reserve` where the size is known or bounded.
|
||||
3. Small-vector types (`smallvec`, fixed inline buffers) for collections
|
||||
that are almost always tiny — confirm the size distribution from the
|
||||
profile or a counting instrument first.
|
||||
4. Arena allocation (`bumpalo`, typed arenas) for phase-scoped object
|
||||
graphs with a clear reset point.
|
||||
|
||||
## Verification notes
|
||||
- Buffer reuse is where stale-data bugs come from: golden replay must
|
||||
cover inputs of *decreasing* size across a session so a dirty buffer
|
||||
would surface. Add such a sequence to the corpus if missing.
|
||||
- Arenas change drop order; targets with observable `Drop` side effects
|
||||
(logging, flushing) can diverge on golden replay — that divergence is
|
||||
a real reject, not noise.
|
||||
|
||||
## Known failure modes
|
||||
- `with_capacity` guesses that overshoot: RSS regression the wall-clock
|
||||
number won't show — check the captured max RSS.
|
||||
- smallvec inline sizes chosen by vibes: an inline buffer bigger than
|
||||
the cache line it saves. Measure, don't guess.
|
||||
- Reuse introducing borrow-checker contortions that force `unsafe` —
|
||||
`unsafe` introduction routes to `needs-human-review` (SPEC §8).
|
||||
@@ -0,0 +1,37 @@
|
||||
# Class 4 — Data layout
|
||||
|
||||
Make the memory hierarchy work for the access pattern.
|
||||
|
||||
## Preconditions
|
||||
- Profile signature: high cache-miss rate in `perf stat` (LLC misses per
|
||||
instruction), hot loops whose flamegraph frames are memory-bound
|
||||
(`perf` cycles concentrated on loads), low IPC (<1) on wide cores.
|
||||
- For false sharing: multithreaded target where per-thread throughput
|
||||
degrades as threads are added; `perf c2c` confirms.
|
||||
|
||||
## Procedure (cheapest first)
|
||||
1. Struct field reordering / packing: kill padding, group hot fields on
|
||||
one cache line, split hot/cold (`#[repr(C)]` review, cold-field
|
||||
box-out).
|
||||
2. AoS→SoA for loops that touch one field across many elements —
|
||||
restructure the container, not the algorithm.
|
||||
3. False-sharing fixes: pad or align per-thread state to 64 bytes
|
||||
(`#[repr(align(64))]`, `crossbeam_utils::CachePadded`).
|
||||
4. Cache blocking / loop tiling for nested loops over large matrices or
|
||||
images; tile to L1/L2, verified empirically, not by datasheet.
|
||||
|
||||
## Verification notes
|
||||
- Layout changes to any type that crosses FFI, serialization, or hashing
|
||||
boundaries can change observable behavior (struct hash, serialized
|
||||
bytes, memcmp-based equality) — golden replay catches this; check the
|
||||
target for `#[repr]`-sensitive traits before patching.
|
||||
- Differential fuzz the functions whose types changed, not just the
|
||||
functions whose code changed.
|
||||
|
||||
## Known failure modes
|
||||
- Reordering fields of a type that derives `Hash`/`Ord` used in ordered
|
||||
iteration → output order changes → replay divergence (correct reject).
|
||||
- AoS→SoA wins the target loop but loses every other access site; the
|
||||
end-to-end number is the verdict, not the microbench.
|
||||
- Alignment padding blowing up per-element size until the working set
|
||||
falls out of cache — the cure worse than the disease.
|
||||
@@ -0,0 +1,43 @@
|
||||
# Class 5 — Algorithmic
|
||||
|
||||
Complexity class and primitive selection. Highest variance: the biggest
|
||||
wins live here, and so do the most seductive wrong ideas.
|
||||
|
||||
## Preconditions
|
||||
- Profile signature: one dominant frame that is *algorithm*, not memory
|
||||
(high IPC, low miss rate, still slow); superlinear scaling of runtime
|
||||
with input size; hot `contains`/`find` over linear structures; repeated
|
||||
recomputation of pure functions on repeated arguments.
|
||||
|
||||
## Procedure (cheapest first)
|
||||
1. Better std/library primitive: `HashMap` for linear scans, `sort_unstable`
|
||||
over `sort`, `entry()` API over get-then-insert, buffered I/O,
|
||||
`memchr`-class scanning.
|
||||
2. Precomputation: hoist invariant work out of loops; build lookup tables
|
||||
at startup when the domain is small.
|
||||
3. Memoization for pure hot functions with high argument repeat rates —
|
||||
confirm the repeat rate from a counting instrument before adding a
|
||||
cache.
|
||||
4. Complexity-class replacement (the real prize): O(n²) → O(n log n)
|
||||
with a known-correct algorithm. Prefer a well-tested library
|
||||
implementation over a hand-rolled one, always.
|
||||
|
||||
## Verification notes
|
||||
- HashMap iteration order is not deterministic order: if results flow
|
||||
from iteration order into output, replay diverges — either sort at the
|
||||
boundary or reject the change.
|
||||
- Memoization + interior mutability in multithreaded targets touches the
|
||||
concurrency review tier (SPEC §8).
|
||||
- Differential fuzz is load-bearing here: algorithmic replacements have
|
||||
edge-case surfaces (empty input, duplicates, already-sorted, unicode)
|
||||
where "equivalent" algorithms differ. 10k iterations minimum, seeded
|
||||
with corpus-derived cases.
|
||||
|
||||
## Known failure modes
|
||||
- Asymptotic win, constant-factor loss at real input sizes — the ledger
|
||||
records the workload; the verdict is measured at that workload.
|
||||
- Caches without bounds: memoization that trades a 10% speedup for
|
||||
unbounded RSS growth. Check max RSS; bound the cache.
|
||||
- "Equivalent" floating-point reformulations that aren't (different
|
||||
rounding paths) — FP output changes route to the tolerance policy or
|
||||
human review, never silently absorbed.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Class 6 — SIMD
|
||||
|
||||
Autovectorization enablement first, intrinsics last (SPEC §4).
|
||||
|
||||
## Preconditions
|
||||
- Profile signature: hot, trip-count-heavy inner loops over contiguous
|
||||
numeric/byte data; scalar math or byte-wise processing in the top
|
||||
frames; disassembly of the hot loop shows scalar ops where vector ops
|
||||
are plausible.
|
||||
|
||||
## Procedure (cheapest first)
|
||||
1. Read the optimizer's mind: check the hot loop's assembly (or
|
||||
`-Cremark`/`-Rpass=loop-vectorize`) for *why* vectorization failed.
|
||||
2. Remove the blocker, in order of cheapness:
|
||||
- Bounds checks: restructure with slices/iterators or hoist a single
|
||||
length check (`&x[..n]` pattern) — not `unsafe` indexing.
|
||||
- Aliasing: split borrows, `split_at_mut`, pass slices instead of
|
||||
overlapping raw pointers.
|
||||
- Loop-carried dependencies: reassociate integer reductions, unroll
|
||||
accumulators.
|
||||
3. Widen the ISA baseline if class 1 hasn't already (`target-cpu`), since
|
||||
SSE2-only baselines strangle the vectorizer.
|
||||
4. `std::simd` / portable-SIMD explicit vectorization.
|
||||
5. Raw intrinsics only when all of the above fail and the win is large;
|
||||
isolate behind a scalar-fallback path selected at runtime.
|
||||
|
||||
## Verification notes
|
||||
- **FP flags are a hard line**: any `-ffast-math`-class flag, FP
|
||||
reassociation, or FMA-contraction change routes to
|
||||
`needs-human-review` — no exceptions, even for a 2x win (SPEC §8).
|
||||
Integer/byte SIMD has no such taint and can auto-accept.
|
||||
- Intrinsics imply `unsafe` → automatic `needs-human-review` + MIRI
|
||||
where feasible + differential fuzz against the scalar path with
|
||||
adversarial lengths (0, 1, lane-1, lane, lane+1, unaligned offsets).
|
||||
|
||||
## Known failure modes
|
||||
- Tail handling: the classic SIMD bug class. Fuzz lengths around lane
|
||||
boundaries explicitly.
|
||||
- Alignment assumptions that hold on the bench box and fault (or crawl)
|
||||
elsewhere.
|
||||
- Runtime feature detection done per-call in the hot loop, eating the
|
||||
win; hoist detection to init.
|
||||
- Integer reduction reassociation is safe; float reduction reassociation
|
||||
changes results — do not confuse the two.
|
||||
@@ -0,0 +1,43 @@
|
||||
# Class 7 — Concurrency
|
||||
|
||||
Last in the order, highest risk. **Every diff in this class routes to
|
||||
`needs-human-review`** (SPEC §8); the agent proposes, a human disposes.
|
||||
|
||||
## Preconditions
|
||||
- Profile signature: multithreaded target where wall time ≫ ideal
|
||||
(cpu-time / cores); lock symbols (`futex`, `pthread_mutex`,
|
||||
`parking_lot`) hot; run-queue latency; per-thread throughput that
|
||||
*drops* as threads are added. coz (Phase 3) is the right prioritizer
|
||||
here — cycle profiles mislead on blocked time.
|
||||
|
||||
## Procedure (cheapest first)
|
||||
1. Shrink critical sections: move allocation, I/O, and computation out
|
||||
from under locks; drop guards early.
|
||||
2. Reduce sharing: per-thread accumulation + merge at the end beats a
|
||||
shared counter; sharding a hot map by key hash beats a fancier lock.
|
||||
3. Right-size the primitive: `RwLock` for read-mostly, `OnceLock` for
|
||||
init-once, atomics for simple counters *with explicitly justified
|
||||
orderings* (start from `SeqCst`; relax only with a written argument).
|
||||
4. Lock-free structures: last resort, library-only (crossbeam), never
|
||||
hand-rolled, and only when the profile proves the lock is the
|
||||
bottleneck after 1–3.
|
||||
|
||||
## Verification notes
|
||||
- TSan gate is mandatory for this class, on both the upstream suite and
|
||||
the golden replay run (SPEC §3.2).
|
||||
- Thread-count sweep: gates and bench at 1, 2, N, and 2N threads —
|
||||
concurrency bugs and wins are both thread-count-dependent.
|
||||
- Nondeterministic output ordering (e.g. results gathered from threads)
|
||||
must be canonicalized at the boundary or the change rejected; replay
|
||||
equivalence is not negotiable to make a parallel win look clean.
|
||||
|
||||
## Known failure modes
|
||||
- Benchmarks too short to hit the contention window: a race that fires
|
||||
once per million ops passes 60s of fuzz. TSan + long soak, and still
|
||||
route to a human.
|
||||
- Relaxed atomics that are correct on x86's strong model and wrong on
|
||||
ARM — orderings need an argument, not a test pass.
|
||||
- Sharding that wins the benchmark's uniform key distribution and loses
|
||||
production's skewed one — holdout workloads (SPEC §11).
|
||||
- Deadlocks introduced by lock-order changes that no gate exercises;
|
||||
the human reviewer checks lock ordering explicitly.
|
||||
@@ -0,0 +1,25 @@
|
||||
# Optimization playbook v0
|
||||
|
||||
Classes are tried in **strict order** (SPEC §4) — cheapest first. Classes
|
||||
1–2 are the "your 30% was a linker flag" tier and are always exhausted
|
||||
before any source change. Each class documents:
|
||||
|
||||
- **Preconditions** — the profile signature that suggests it applies
|
||||
- **Procedure** — what to actually try, cheapest variant first
|
||||
- **Verification notes** — gate interactions specific to this class
|
||||
- **Known failure modes** — where this class produces fake wins or real bugs
|
||||
|
||||
| # | Class | Touches source? | Human review? |
|
||||
|---|-------|-----------------|---------------|
|
||||
| 1 | [Build configuration](01-build-config.md) | no | only PGO/BOLT pipeline changes |
|
||||
| 2 | [Allocator swap](02-allocator.md) | link line only | no |
|
||||
| 3 | [Allocation churn](03-allocation-churn.md) | yes | no |
|
||||
| 4 | [Data layout](04-data-layout.md) | yes | no |
|
||||
| 5 | [Algorithmic](05-algorithmic.md) | yes | no |
|
||||
| 6 | [SIMD](06-simd.md) | yes | FP-flag changes: always |
|
||||
| 7 | [Concurrency](07-concurrency.md) | yes | **always** |
|
||||
|
||||
The agent's prompting spine (SPEC §3.5): state the hypothesis before
|
||||
patching; prefer the cheapest untried class; max two iterations per
|
||||
rejected hypothesis; a rejection with a clean ledger row is a successful
|
||||
outcome.
|
||||
@@ -0,0 +1,13 @@
|
||||
# results/calibration/
|
||||
|
||||
Evidence for the bench-runner acceptance criteria (SPEC §3.1):
|
||||
|
||||
- **A/A self-tests**: same binary as both sides must yield a null
|
||||
verdict; false-positive rate < 5% across sessions.
|
||||
- **Regression injection**: a synthetic 5% slowdown must be detected
|
||||
≥ 95% of the time.
|
||||
|
||||
One JSON file per calibration session, including the full env
|
||||
fingerprint. These files are the license to trust every number the
|
||||
system reports afterward; they are referenced from case studies and ROI
|
||||
reports.
|
||||
@@ -0,0 +1,34 @@
|
||||
# targets/
|
||||
|
||||
One directory per optimization target. This is the **only**
|
||||
agent-writable path in the repository (SPEC §10); everything else is
|
||||
trust layer.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
targets/<name>/
|
||||
target.toml # upstream repo URL, pinned commit, build & bench commands
|
||||
equivalence.toml # optional; absent = byte-identical required (SPEC §3.2)
|
||||
workspace/ # agent's checkout — patches land here via harness git-apply
|
||||
```
|
||||
|
||||
Selection criteria (SPEC §6): CPU-bound (>70% user time), meaningful
|
||||
test suite, scriptable workload, permissive license, active but not
|
||||
hyper-optimized, builds cleanly in a container. Good hunting: codecs,
|
||||
parsers, serializers, image processing, compression, static-site
|
||||
generators, linters.
|
||||
|
||||
## equivalence.toml
|
||||
|
||||
Default (no file) is byte-identical outputs. FP-producing targets may
|
||||
declare an explicit tolerance policy:
|
||||
|
||||
```toml
|
||||
mode = "fp-tolerance"
|
||||
abs = 1e-9 # max absolute difference per numeric value
|
||||
rel = 1e-6 # max relative difference per numeric value
|
||||
```
|
||||
|
||||
Declaring a tolerance covers output comparison only; FP *flag* changes
|
||||
still route to needs-human-review (SPEC §8).
|
||||
Reference in New Issue
Block a user