mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
benbrownandClaude Opus 4.8
899d55f4b6
An adversarial audit found the enforcement story ahead of the code: gates that were documented as hard gates were advisory, manual, or absent, and several human-facing claims outran the machine record. This makes the claims true. High findings - Differential fuzz was documentation, not enforcement: verdict hardcoded fuzz_iters=0 and diff-test hardcoded the gate to Skipped, while the case studies claimed 10k-iteration fuzz on the same wins. Fuzz is now a real pipeline gate (strict FUZZ-RESULT contract, executed count recorded), and it is inherently baseline-vs-candidate, so it runs on the accept path and reports Skipped in the standalone `just gates` flow. New hard rule: a machine `accepted` verdict is impossible unless the fuzz gate actually passed — otherwise it caps at needs-human-review. - The agent's system prompt claimed FP/concurrency/unsafe patches were "routed to human review automatically". No such mechanism existed. Added a conservative lexical risk classifier (crates/verdict/src/risk.rs) that forces needs-human-review on any would-be accept carrying those signals; fp-tolerance equivalence always signals. - The local verdict path ran agent-influenced code unsandboxed. It now runs under scripts/no-net.sh (network namespace) and fails closed. Upstream test suites are hash-verified against corpora/<t>/TESTSUITE.sha256 when present (scripts/pin-testsuite.sh generates them). Also fixed, found while fixing the above - The pipeline script's `set -e` was inert (brace group as the left operand of `&&`), so a failing candidate build did not abort the run and verdict could bench a stale binary. Now a scoped subshell whose status is branched on at top level. - A wrapper that exits before exec'ing (no-net.sh refusing, bad wrapper path) stranded RUNNING.lock and left read_verdict polling forever. Failure accounting now lives outside the wrapper. - harnessd's patch allowlist ignored git rename/copy headers and --no-prefix diffs; the rename-escape case is now a committed regression test. - The workspace did not compile on Windows (ungated Unix APIs). cfg-gated; 71 portable tests now run on both platforms. - pin-check regenerated the corpus manifest instead of verifying it; it is now verify-only, with a deliberate `just pin-corpus` to re-pin. - Max RSS is captured per run on Unix; PMU counters are opt-in diagnostics (compare --perf-stat), never part of the accept decision. Docs now match the code, including what is NOT yet enforced: no target ships a TESTSUITE.sha256 pin, no target declares [build].tsan, and matmul is not wired to the fp-tolerance gate — each is stated as mechanized-but-inert rather than done. Corrected the stale "33 ledger rows" (34), the playbook's class-3 cJSON win (rejected), adversarial-review.md's sanitizer-separation claim (five accepted rows carry sanitizers_clean=false), the unsupported "40,000 requests" figure, and the blanket "historical rows record fuzz_iters=0" (8 of 10; phase0-tokei-002/-003 record 4,332). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
51 lines
2.8 KiB
TOML
51 lines
2.8 KiB
TOML
# cJSON — ultralightweight JSON parser/printer (C)
|
|
# Selection vs SPEC §6: CPU-bound parse+serialize; hermetic Unity/CTest
|
|
# suite vendored in-tree; deterministic scripted workload; MIT license;
|
|
# active but deliberately "simple", no dedicated perf team — the richer
|
|
# legacy-C hunting ground Phase 3 calls for.
|
|
|
|
[source]
|
|
repo = "https://github.com/DaveGamble/cJSON.git"
|
|
commit = "c859b25da02955fef659d658b8f324b5cde87be3" # v1.7.19
|
|
license = "MIT"
|
|
|
|
[fetch]
|
|
# targets/*/workspace is gitignored; reproduce it with:
|
|
# git clone https://github.com/DaveGamble/cJSON.git workspace
|
|
# git -C workspace checkout v1.7.19
|
|
# The bench/golden driver targets/cjson/harness.c and the Unity suite in
|
|
# workspace/tests/ (unity vendored in-tree) are hermetic — no submodules.
|
|
|
|
[build]
|
|
# Compile the trust-layer harness against the (patched) workspace cJSON.c.
|
|
# {out} is the isolation dir; the agent may patch only workspace/*, never
|
|
# harness.c, so the measurement rig is fixed across baseline/candidate.
|
|
baseline = "mkdir -p {out} && clang -O2 -DNDEBUG -I targets/cjson/workspace -o {out}/cjson-bench targets/cjson/harness.c targets/cjson/workspace/cJSON.c"
|
|
binary = "{out}/cjson-bench"
|
|
# ASan + UBSan (C/C++ analogue of the Rust nightly ASan path); run on
|
|
# every would-be accept (SPEC §8), halting on any leak/UB.
|
|
sanitizer = "mkdir -p {out} && clang -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -I targets/cjson/workspace -o {out}/cjson-asan targets/cjson/harness.c targets/cjson/workspace/cJSON.c"
|
|
sanitizer_binary = "{out}/cjson-asan"
|
|
|
|
[bench]
|
|
command = "{binary} corpora/cjson/input/big.json 2 > /dev/null"
|
|
workload = "5.6 MB synthetic JSON (20k records: nested objects/arrays, escaped/unicode strings, ints, dense floats), parsed+serialized 2x, single thread"
|
|
|
|
[gates]
|
|
# Upstream Unity suite via CTest — parse_number/string/array/object,
|
|
# print_*, misc_tests, compare/minify. Tests the PATCHED cJSON.c.
|
|
tests = "cmake -S targets/cjson/workspace -B targets/cjson/testbuild -DENABLE_CJSON_TEST=On -DCMAKE_BUILD_TYPE=Debug >/dev/null 2>&1 && cmake --build targets/cjson/testbuild -j >/dev/null 2>&1 && ctest --test-dir targets/cjson/testbuild --output-on-failure"
|
|
# Golden: full re-serialization + checksum of the corpus, byte-hashed.
|
|
golden = "{binary} corpora/cjson/input/big.json 1"
|
|
# Differential fuzz, pipeline-integrated: diff-test substitutes {iters},
|
|
# {baseline} (the pristine binary the accept path rebuilds) and {candidate},
|
|
# then parses the script's final FUZZ-RESULT line. The harness owns where the
|
|
# pristine build lands, so the script takes the baseline from argv and never
|
|
# hardcodes a path; paths may be repo-root-relative or absolute.
|
|
fuzz = "python3 scripts/diff-fuzz-cjson.py targets/cjson/fuzz-work {iters} {baseline} {candidate}"
|
|
|
|
[corpus]
|
|
manifest = "corpora/cjson/MANIFEST.sha256"
|
|
root = "corpora/cjson/input"
|
|
golden_sha256 = "corpora/cjson/GOLDEN.sha256"
|