mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
Claude
a78692bbb7
- bench-runner service mode (SPEC §3.1 mode c): a coordinated-omission- correct, open-loop, fixed-rate load generator; latency measured from the intended send time so queueing is counted, not hidden. Interleaved A/B of two server binaries, exact percentiles (bounded sample, no HDR bucketing error), bootstrap p50/p99 CIs via the existing ratio-CI machine. service-calibrate: A/A false-positive + injected latency- regression detection (calibration in progress). - targets/cjson/service.c: minimal single-threaded HTTP server wrapping the patched cJSON.c (trust-layer, outside the workspace allowlist); env-toggled per-request busy-wait for the injection calibration only. - report generator (crates/report bin): reads a ledger row + optional service-latency JSON, renders throughput→cores→dollars and/or latency percentiles, every figure with its CI, pricing source, and the methodology printed inline (SPEC §9); flags any non-accepted row so a rejected change never reads as a committed saving. - ledger::report_row; justfile service/service-calibrate/report. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
87 lines
3.5 KiB
Makefile
87 lines
3.5 KiB
Makefile
# hotpath task runner. `just --list` for a summary.
|
|
|
|
# Build the trust-layer workspace
|
|
build:
|
|
cargo build --workspace
|
|
|
|
# Run all trust-layer unit tests
|
|
test:
|
|
cargo test --workspace
|
|
|
|
# Lint: warnings are errors in the trust layer
|
|
lint:
|
|
cargo clippy --workspace --all-targets -- -D warnings
|
|
cargo fmt --all --check
|
|
|
|
# A/A self-test of the bench harness against a sample command.
|
|
# Must print a null verdict; an "accept" is a calibration failure.
|
|
aa cmd="sleep 0.05":
|
|
cargo run -p bench-runner -- --config config/accept.toml aa --cmd "{{cmd}}"
|
|
|
|
# Interleaved A/B comparison of two shell commands
|
|
compare baseline candidate:
|
|
cargo run -p bench-runner -- --config config/accept.toml compare \
|
|
--baseline "{{baseline}}" --candidate "{{candidate}}"
|
|
|
|
# Verify a target corpus against its pinned manifest
|
|
# (wired through diff-test CLI once it exists; manual check for now)
|
|
pin-check target:
|
|
cd corpora/{{target}} && sh gen-corpus.sh
|
|
|
|
# Automated calibration: N A/A sessions (<5% false-positive required) +
|
|
# N injected-slowdown sessions (>=95% detection required), JSON evidence
|
|
calibrate cmd out sessions="20":
|
|
cargo run -p bench-runner -- --config config/accept.toml calibrate \
|
|
--cmd "{{cmd}}" --sessions {{sessions}} --out "{{out}}"
|
|
|
|
# Equivalence gates for a target: corpus pin (refuses on mismatch) ->
|
|
# upstream tests -> golden replay. Fuzz/sanitizers stay per-attempt.
|
|
gates target:
|
|
cargo run -p diff-test -- {{target}}
|
|
|
|
# One-shot attempt verdict: gates on candidate -> interleaved A/B vs a
|
|
# pristine-rebuilt baseline -> ledger row. Extra flags pass through
|
|
# (--patch-file, --needs-human-review, --baseline-bin).
|
|
verdict target candidate run_id class hypothesis hotspot *flags:
|
|
cargo run -p verdict -- {{target}} --rebuild-baseline \
|
|
--candidate-bin "{{candidate}}" --run-id "{{run_id}}" \
|
|
--playbook-class {{class}} --hypothesis "{{hypothesis}}" \
|
|
--hotspot "{{hotspot}}" {{flags}}
|
|
|
|
# One unattended agent attempt behind the SPEC §10 OS boundary:
|
|
# harnessd (trusted uid) on a Unix socket, agent loop as user hpagent.
|
|
agent-attempt target run_id max_turns="40":
|
|
sh scripts/agent-isolated.sh {{target}} {{run_id}} {{max_turns}}
|
|
|
|
# Verify the OS boundary from the agent user's side (negative + positive)
|
|
isolation-check:
|
|
sh scripts/isolation-check.sh
|
|
|
|
# Install the coz causal profiler (apt, else source build)
|
|
install-coz:
|
|
sh scripts/install-coz.sh
|
|
|
|
# Causal profile of a C/C++ target: where a speedup would raise throughput
|
|
coz target iters="200":
|
|
sh scripts/coz-profile.sh {{target}} {{iters}}
|
|
python3 scripts/coz-summary.py results/{{target}}/coz/profile.coz
|
|
|
|
# Service mode (Phase 4): interleaved A/B latency bench of two server
|
|
# binaries under coordinated-omission-correct open-loop load. p50/p99 CIs.
|
|
service baseline candidate doc *flags:
|
|
cargo run -p bench-runner -- --config config/accept.toml service \
|
|
--baseline-bin "{{baseline}}" --candidate-bin "{{candidate}}" \
|
|
--doc "{{doc}}" --pin "taskset -c 2" {{flags}}
|
|
|
|
# Service-mode calibration: A/A false-positive + injected latency-regression
|
|
# detection (SPEC §3.1); writes JSON evidence.
|
|
service-calibrate server doc out *flags:
|
|
cargo run -p bench-runner -- --config config/accept.toml service-calibrate \
|
|
--server-bin "{{server}}" --doc "{{doc}}" --pin "taskset -c 2" \
|
|
--out "{{out}}" {{flags}}
|
|
|
|
# Mechanical ROI report from a ledger row (SPEC §9): throughput→cores→
|
|
# dollars and/or latency percentiles, CIs + methodology printed inline.
|
|
report run_id *flags:
|
|
cargo run -p report -- --run-id "{{run_id}}" {{flags}}
|