diff --git a/.gitignore b/.gitignore index eab05b6..168be9c 100644 --- a/.gitignore +++ b/.gitignore @@ -239,3 +239,7 @@ targets/*/baseline/ # Phase 2 agent artifacts (transcripts kept; pending logs are scratch) results/pending/ agent/**/__pycache__/ + +# cjson corpus is regenerable via gen-corpus.sh (deterministic) +corpora/cjson/input/ +targets/cjson/testbuild/ diff --git a/corpora/cjson/GOLDEN.sha256 b/corpora/cjson/GOLDEN.sha256 new file mode 100644 index 0000000..8053657 --- /dev/null +++ b/corpora/cjson/GOLDEN.sha256 @@ -0,0 +1 @@ +9273ad45a9aa51dd7f941d938a771de3e95a7b3ac807a880f24392c1eb6a72bb golden-stdout diff --git a/corpora/cjson/MANIFEST.sha256 b/corpora/cjson/MANIFEST.sha256 new file mode 100644 index 0000000..39af4dd --- /dev/null +++ b/corpora/cjson/MANIFEST.sha256 @@ -0,0 +1 @@ +3b0c1dee5a75de59b12e692780896f509c93f3c6b8da7cc337ae4df7fbf70075 ./big.json diff --git a/corpora/cjson/gen-corpus.sh b/corpora/cjson/gen-corpus.sh new file mode 100644 index 0000000..3763094 --- /dev/null +++ b/corpora/cjson/gen-corpus.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Deterministically generate the cjson corpus — a large synthetic JSON +# document exercising every parse path: nested objects/arrays, strings +# with escapes and unicode, ints, floats (the strtod/sprintf hot path), +# booleans, nulls. No randomness — byte-identical on every machine, so +# the MANIFEST.sha256 pin is reproducible. Run from repo root: +# sh corpora/cjson/gen-corpus.sh +set -eu +DIR="$(cd "$(dirname "$0")" && pwd)" +OUT="$DIR/input/big.json" +mkdir -p "$DIR/input" + +python3 - "$OUT" <<'PY' +import json, sys + +# Deterministic: a fixed linear-congruential stream, no `random` module, +# so output is identical everywhere. +class Det: + def __init__(self, seed=2463534242): + self.s = seed + def next(self): + self.s ^= (self.s << 13) & 0xFFFFFFFF + self.s ^= (self.s >> 17) + self.s ^= (self.s << 5) & 0xFFFFFFFF + return self.s & 0xFFFFFFFF + def rng(self, n): + return self.next() % n + +d = Det() +WORDS = ["alpha","bravo","charlie","delta","echo","foxtrot","golf","hotel", + "quote\"inside","tab\tchar","new\nline","back\\slash","unécode","☃snow"] + +def record(i): + return { + "id": i, + "guid": f"{d.next():08x}-{d.next():04x}", + "active": bool(d.rng(2)), + "score": d.rng(1000000) / 1000.0, # float -> strtod/sprintf path + "ratio": d.rng(1 << 30) / (1 << 30), # dense fractional float + "count": d.rng(1 << 31) - (1 << 30), # signed int + "name": WORDS[d.rng(len(WORDS))] + str(d.rng(9999)), + "tags": [WORDS[d.rng(len(WORDS))] for _ in range(d.rng(6))], + "coords": [d.rng(1 << 20) / 997.0 for _ in range(3)], + "meta": None if d.rng(4) == 0 else { + "created": d.next(), + "nested": {"a": d.rng(100), "b": [d.rng(50) for _ in range(d.rng(4))]}, + }, + } + +doc = { + "schema": "hotpath.cjson.corpus/v1", + "generated_by": "gen-corpus.sh (deterministic)", + "records": [record(i) for i in range(20000)], +} +with open(sys.argv[1], "w") as f: + json.dump(doc, f, ensure_ascii=False, separators=(",", ":")) +PY + +echo "wrote $OUT ($(wc -c < "$OUT") bytes)" +# Pin the corpus and regenerate the manifest. +( cd "$DIR/input" && find . -type f -print0 | sort -z | xargs -0 sha256sum ) > "$DIR/MANIFEST.sha256" +echo "manifest: $DIR/MANIFEST.sha256" diff --git a/crates/diff-test/src/target.rs b/crates/diff-test/src/target.rs index 0b7a4f4..2596358 100644 --- a/crates/diff-test/src/target.rs +++ b/crates/diff-test/src/target.rs @@ -33,10 +33,31 @@ pub struct Source { #[derive(Debug, Deserialize)] pub struct Build { - /// Baseline build command, run from repo root. + /// Build command, run from repo root. `{out}` is substituted with an + /// isolation directory the harness provides (per-baseline, per- + /// candidate, per-sanitizer) — the command MUST place all artifacts + /// under it. A cargo target sets `CARGO_TARGET_DIR={out}`; a make + /// target builds and copies the binary to `{out}`. Language-agnostic: + /// nothing here assumes cargo. pub baseline: String, - /// Built binary path, relative to repo root. + /// Built binary path with `{out}` substituted — where `baseline` + /// leaves the binary given an isolation dir. pub binary: String, + /// ASan+LSan (or ASan+UBSan for C/C++) build command; `{out}` + /// substituted. Run on every would-be accept (SPEC §8). If absent, + /// an accept cannot be sanitizer-verified and is capped at + /// needs-human-review. + #[serde(default)] + pub sanitizer: Option, + /// Sanitizer binary path with `{out}` substituted. + #[serde(default)] + pub sanitizer_binary: Option, +} + +/// Substitute the `{out}` isolation-dir placeholder in a build command or +/// binary-path template. +pub fn subst_out(template: &str, out_dir: &str) -> String { + template.replace("{out}", out_dir) } #[derive(Debug, Deserialize)] diff --git a/crates/harnessd/src/main.rs b/crates/harnessd/src/main.rs index 7192b39..d7638f8 100644 --- a/crates/harnessd/src/main.rs +++ b/crates/harnessd/src/main.rs @@ -339,29 +339,30 @@ fn handle(root: &Path, line: &str) -> Result { )?)?; let t = pending["target"].as_str().unwrap(); let spec = diff_test::target::TargetSpec::load(root, t)?; - let bin = Path::new(&spec.build.binary) - .file_name() - .unwrap() - .to_str() - .unwrap(); let diff_path = root.join(format!("results/pending/{patch_id}.json.diff")); std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?; // The build + verdict pipeline runs for minutes — far past the MCP // transport's per-call cap. Launch it detached, writing progress to // a log; the agent observes the result via the read_verdict op, // which reads the append-only ledger row once written. - let cand_dir = format!("targets/{t}/candidate-{patch_id}"); - let cand_bin = format!("{cand_dir}/release/{bin}"); + // {out} isolates the candidate build (cargo: CARGO_TARGET_DIR; + // make: build+copy into it) — language-agnostic, mirrors the + // baseline rebuild in verdict. + let cand_dir = root + .join(format!("targets/{t}/candidate-{patch_id}")) + .to_string_lossy() + .into_owned(); + let cand_build = diff_test::target::subst_out(&spec.build.baseline, &cand_dir); + let cand_bin = diff_test::target::subst_out(&spec.build.binary, &cand_dir); let log = format!("results/pending/{run_id}.log"); // On any failure (candidate build, gates crash) the trap line // marks the log so read_verdict can report "failed" instead of // leaving the agent polling a verdict that will never exist. let script = format!( - "{{ set -e\nCARGO_TARGET_DIR={cand} {build}\ncargo run -q -p verdict -- {tgt} \ + "{{ set -e\n{build}\ncargo run -q -p verdict -- {tgt} \ --rebuild-baseline --candidate-bin {cbin} --run-id {rid} --playbook-class {cls} \ --hypothesis {hyp} --hotspot {hs} --patch-file {pf}\n}} || echo HOTPATH-PIPELINE-FAILED\n", - cand = shq(&cand_dir), - build = spec.build.baseline, + build = cand_build, tgt = shq(t), cbin = shq(&cand_bin), rid = shq(run_id), diff --git a/crates/verdict/src/main.rs b/crates/verdict/src/main.rs index fc37c97..84dd7ab 100644 --- a/crates/verdict/src/main.rs +++ b/crates/verdict/src/main.rs @@ -79,14 +79,14 @@ fn rebuild_pristine_baseline( "git stash failed" ); } + let out_dir = root + .join(format!("targets/{target}/baseline")) + .to_string_lossy() + .into_owned(); let build = std::process::Command::new("sh") .arg("-c") - .arg(&spec.build.baseline) + .arg(diff_test::target::subst_out(&spec.build.baseline, &out_dir)) .current_dir(root) - .env( - "CARGO_TARGET_DIR", - root.join(format!("targets/{target}/baseline")), - ) .status(); if dirty { anyhow::ensure!( @@ -98,13 +98,9 @@ fn rebuild_pristine_baseline( ); } anyhow::ensure!(build?.success(), "baseline build failed"); - let bin_name = std::path::Path::new(&spec.build.binary) - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?; - let baseline = format!("targets/{target}/baseline/release/{bin_name}"); + let baseline = diff_test::target::subst_out(&spec.build.binary, &out_dir); anyhow::ensure!( - root.join(&baseline).exists(), + std::path::Path::new(&baseline).exists(), "baseline binary missing: {baseline}" ); println!("baseline rebuilt from pristine checkout: {baseline}"); @@ -117,40 +113,43 @@ fn rebuild_pristine_baseline( /// were per-attempt manual) and the human audit had to overturn it — the /// accept path now runs the check itself and caps flagged wins at /// needs-human-review. Rejections skip it: they ship nothing. -fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result { - let ws = root.join(format!("targets/{target}/workspace")); - let asan_dir = root.join(format!("targets/{target}/asan")); - println!("sanitizers: ASan+LSan build (nightly) of the patched tree…"); - let status = std::process::Command::new("cargo") - .args([ - "+nightly", - "build", - "-q", - "--target", - "x86_64-unknown-linux-gnu", - ]) - .env("RUSTFLAGS", "-Zsanitizer=address") - .env("CARGO_TARGET_DIR", &asan_dir) - .current_dir(&ws) +/// Returns Some(clean) if the sanitizer gate ran, or None if the target +/// declares no sanitizer build (an accept then cannot be verified and is +/// capped at needs-human-review by the caller). +fn sanitizer_check( + root: &std::path::Path, + target: &str, + spec: &TargetSpec, +) -> Result> { + let (Some(build_tpl), Some(bin_tpl)) = (&spec.build.sanitizer, &spec.build.sanitizer_binary) + else { + println!("sanitizers: target declares no sanitizer build — cannot verify an accept"); + return Ok(None); + }; + let out_dir = root + .join(format!("targets/{target}/asan")) + .to_string_lossy() + .into_owned(); + println!("sanitizers: ASan build of the patched tree…"); + let status = std::process::Command::new("sh") + .arg("-c") + .arg(diff_test::target::subst_out(build_tpl, &out_dir)) + .current_dir(root) .status()?; anyhow::ensure!(status.success(), "sanitizer build failed"); - let bin_name = std::path::Path::new(&spec.build.binary) - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?; - let asan_bin = asan_dir.join(format!("x86_64-unknown-linux-gnu/debug/{bin_name}")); - let cmd = spec - .bench - .command - .replace("{binary}", asan_bin.to_str().unwrap()); + let asan_bin = diff_test::target::subst_out(bin_tpl, &out_dir); + let cmd = spec.bench.command.replace("{binary}", &asan_bin); let out = std::process::Command::new("sh") .arg("-c") .arg(&cmd) + // detect_leaks is a no-op for the C/C++ UBSan combo but harmless. .env("ASAN_OPTIONS", "detect_leaks=1") + .env("UBSAN_OPTIONS", "halt_on_error=1:print_stacktrace=1") .current_dir(root) .output()?; let stderr = String::from_utf8_lossy(&out.stderr); - let clean = out.status.success() && !stderr.contains("ERROR:"); + let clean = + out.status.success() && !stderr.contains("ERROR:") && !stderr.contains("runtime error:"); if clean { println!("sanitizers: clean"); } else { @@ -161,7 +160,7 @@ fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> R println!(" {l}"); } } - Ok(clean) + Ok(Some(clean)) } fn now_utc() -> String { @@ -265,11 +264,18 @@ fn main() -> Result<()> { v = Verdict::NeedsHumanReview; } if v == Verdict::Accepted { - let clean = sanitizer_check(&root, &cli.target, &spec)?; - gate_results.sanitizers_clean = clean; - if !clean { - println!("verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)"); - v = Verdict::NeedsHumanReview; + match sanitizer_check(&root, &cli.target, &spec)? { + Some(true) => gate_results.sanitizers_clean = true, + Some(false) => { + println!( + "verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)" + ); + v = Verdict::NeedsHumanReview; + } + None => { + println!("verdict: no sanitizer build declared; capping accept at needs-human-review (SPEC §8)"); + v = Verdict::NeedsHumanReview; + } } } let fp = EnvFingerprint::collect(pin, "system-default"); diff --git a/justfile b/justfile index 140fe98..5f86829 100644 --- a/justfile +++ b/justfile @@ -56,3 +56,12 @@ agent-attempt target run_id max_turns="40": # Verify the OS boundary from the agent user's side (negative + positive) isolation-check: sh scripts/isolation-check.sh + +# Install the coz causal profiler (apt, else source build) +install-coz: + sh scripts/install-coz.sh + +# Causal profile of a C/C++ target: where a speedup would raise throughput +coz target iters="200": + sh scripts/coz-profile.sh {{target}} {{iters}} + python3 scripts/coz-summary.py results/{{target}}/coz/profile.coz diff --git a/results/ledger.sqlite b/results/ledger.sqlite index 6b1ee41..330c371 100644 Binary files a/results/ledger.sqlite and b/results/ledger.sqlite differ diff --git a/scripts/coz-profile.sh b/scripts/coz-profile.sh new file mode 100644 index 0000000..8837de0 --- /dev/null +++ b/scripts/coz-profile.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# coz causal profiling for a C/C++ target (SPEC §12). Unlike a flat +# profiler (callgrind ranks where time is SPENT), coz measures where a +# speedup would actually RAISE throughput — it virtually speeds up each +# line and reports the causal effect on the harness's progress points. +# Use it to rank playbook targets when the flat profile is ambiguous. +# +# scripts/coz-profile.sh cjson [iters] +# +# Requires: coz on PATH (build: git clone github.com/plasma-umass/coz && +# see scripts/install-coz.sh) and the target's harness built with +# -DHOTPATH_COZ (this script does that). Output: results//coz/ +set -eu +ROOT=$(cd "$(dirname "$0")/.." && pwd) +TARGET=${1:?usage: coz-profile.sh [iters]} +ITERS=${2:-200} +OUT="$ROOT/results/$TARGET/coz" +mkdir -p "$OUT" + +command -v coz >/dev/null 2>&1 || { + echo "coz not on PATH — run scripts/install-coz.sh first" >&2 + exit 1 +} + +# coz needs debug line info and the progress point compiled in. +BIN="$OUT/cjson-bench-coz" +clang -O2 -g -gdwarf-4 -DHOTPATH_COZ \ + -I "$ROOT/targets/$TARGET/workspace" \ + -I "$(dirname "$(command -v coz)")/../include" \ + -o "$BIN" \ + "$ROOT/targets/$TARGET/harness.c" \ + "$ROOT/targets/$TARGET/workspace/cJSON.c" \ + -ldl + +cd "$ROOT" +coz run --output "$OUT/profile.coz" --- \ + "$BIN" corpora/$TARGET/input/big.json "$ITERS" >/dev/null + +echo "coz profile written: $OUT/profile.coz" +echo "view at https://plasma-umass.org/coz/ (load the .coz file) or:" +echo " python3 scripts/coz-summary.py $OUT/profile.coz" diff --git a/scripts/coz-summary.py b/scripts/coz-summary.py new file mode 100644 index 0000000..b80d242 --- /dev/null +++ b/scripts/coz-summary.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Summarize a coz causal profile: rank source lines by their estimated +causal effect on throughput. coz writes newline-delimited JSON records; +for each (line, virtual-speedup) it records progress-point throughput. +We fit the slope of throughput-gain vs virtual-speedup per line — a +steeper positive slope means "speeding up this line most raises overall +throughput", which is exactly the class-selection signal a flat profiler +can't give. + + python3 scripts/coz-summary.py results//coz/profile.coz +""" + +from __future__ import annotations + +import collections +import json +import sys +from pathlib import Path + + +def main() -> int: + path = Path(sys.argv[1]) if len(sys.argv) > 1 else None + if not path or not path.exists(): + print("usage: coz-summary.py ", file=sys.stderr) + return 2 + + # experiments: line -> list of (speedup, progress-rate) + exps: dict[str, list[tuple[float, float]]] = collections.defaultdict(list) + baseline: dict[str, float] = {} + for raw in path.read_text().splitlines(): + raw = raw.strip() + if not raw: + continue + try: + rec = json.loads(raw) + except json.JSONDecodeError: + continue + if rec.get("type") != "experiment": + continue + line = rec.get("selected", "?") + speedup = float(rec.get("speedup", 0.0)) + duration = float(rec.get("duration", 0.0)) + delta = float(rec.get("delta", 0.0)) + if delta <= 0: + continue + rate = duration / delta # time per progress unit; lower = faster + exps[line].append((speedup, rate)) + if speedup == 0.0: + baseline[line] = rate + + rows = [] + for line, pts in exps.items(): + base = baseline.get(line) + if not base or len(pts) < 2: + continue + # % throughput improvement at the max virtual speedup tried + pts.sort() + s_max, r_max = pts[-1] + if s_max <= 0 or r_max <= 0: + continue + gain = (base - r_max) / base # fractional throughput gain + rows.append((gain, s_max, line)) + + rows.sort(reverse=True) + print(f"{'throughput_gain':>16} {'@speedup':>9} line") + print("-" * 70) + for gain, s_max, line in rows[:25]: + print(f"{gain * 100:>14.1f}% {s_max * 100:>7.0f}% {line}") + if not rows: + print("(no usable experiments — did the run mark progress points?)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/diff-fuzz-cjson.py b/scripts/diff-fuzz-cjson.py new file mode 100644 index 0000000..4a6b2c3 --- /dev/null +++ b/scripts/diff-fuzz-cjson.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Differential fuzz for cjson attempts: pristine vs candidate harness must +produce byte-identical stdout (serialization + checksum) AND identical exit +code on mutated JSON. cJSON is a deterministic single-threaded parser, so +raw byte comparison needs no canonicalization. + + python3 scripts/diff-fuzz-cjson.py + +Seeds: slices of the pinned corpus plus hand JSON fragments. Mutations +target parser edges: structural tokens, string escapes, number forms +(leading zeros, +/- exponents, huge/deep values), and raw byte noise. +""" + +import random +import subprocess +import sys +from pathlib import Path + +ROOT = Path("/home/user/peltier.io") +BASE = ROOT / "targets/cjson/baseline/cjson-bench" +CAND = ROOT / sys.argv[3] if len(sys.argv) > 3 else ROOT / "targets/cjson/baseline/cjson-bench" +CORPUS = ROOT / "corpora/cjson/input/big.json" +WORK = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/tmp/difffuzz-cjson") +ITERS = int(sys.argv[2]) if len(sys.argv) > 2 else 10000 + +TOKENS = [ + b"{", b"}", b"[", b"]", b'"', b":", b",", b"\\", b"\\n", b"\\u00e9", b"\\uD83D", + b"true", b"false", b"null", b"0", b"-0", b"1e999", b"-1.5E-3", b"00", b"1.", + b".5", b"1e", b"NaN", b"Infinity", b'\\"', b'"key"', b"[]", b"{}", b'""', + b"1234567890123456789012345", b"\x00", b"\xff", b"\t", b"\n", +] + +SEEDS_STATIC = [ + b'{}', b'[]', b'{"a":1}', b'[1,2,3]', b'"hello"', b'true', b'null', + b'{"n":-1.5e10,"s":"a\\tb","u":"\\u00e9","arr":[null,false,{}]}', + b'[[[[[[1]]]]]]', b'{"a":{"b":{"c":{"d":1}}}}', b'123.456e-78', + b'{"esc":"\\"\\\\\\/\\b\\f\\n\\r\\t"}', b' { "x" : [ 1 , 2 ] } ', +] + + +def mutate(data: bytes, rng: random.Random) -> bytes: + b = bytearray(data) + for _ in range(rng.randrange(1, 8)): + if not b: + b = bytearray(rng.choice(TOKENS)) + continue + op = rng.randrange(5) + pos = rng.randrange(len(b) + 1) + if op == 0: + b[pos:pos] = rng.choice(TOKENS) + elif op == 1 and len(b) > 2: + del b[pos:min(len(b), pos + rng.randrange(1, 12))] + elif op == 2 and pos < len(b): + b[pos] = rng.randrange(256) + elif op == 3: + end = min(len(b), pos + rng.randrange(1, 48)) + b[end:end] = b[pos:end] + else: + b[pos:pos] = rng.choice(TOKENS) + return bytes(b) + + +def run(binary: Path, path: Path) -> tuple[bytes, int]: + r = subprocess.run([str(binary), str(path), "1"], capture_output=True, timeout=30) + return r.stdout, r.returncode + + +def main() -> int: + rng = random.Random(0x1CE_B00C) + corpus = CORPUS.read_bytes() + seeds = list(SEEDS_STATIC) + # windows of the real corpus (often invalid mid-slice — exercises the + # error path, which must diverge identically or not at all) + for o in range(0, len(corpus) - 4096, len(corpus) // 200): + for n in (64, 512, 4096): + seeds.append(corpus[o : o + n]) + WORK.mkdir(parents=True, exist_ok=True) + case = WORK / "case.json" + divergences = 0 + for i in range(ITERS): + payload = mutate(rng.choice(seeds), rng) + case.write_bytes(payload) + a_out, a_rc = run(BASE, case) + b_out, b_rc = run(CAND, case) + if a_out != b_out or a_rc != b_rc: + divergences += 1 + keep = WORK / f"divergence-{i}" + keep.mkdir(exist_ok=True) + (keep / "input.json").write_bytes(payload) + (keep / "baseline.out").write_bytes(a_out + f"\nrc={a_rc}".encode()) + (keep / "candidate.out").write_bytes(b_out + f"\nrc={b_rc}".encode()) + print(f"DIVERGENCE at iter {i} (rc {a_rc} vs {b_rc}); kept in {keep}") + if (i + 1) % 1000 == 0: + print(f"{i + 1}/{ITERS}, {divergences} divergences", flush=True) + print(f"done: {ITERS} inputs, {divergences} divergences") + return 1 if divergences else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/install-coz.sh b/scripts/install-coz.sh new file mode 100644 index 0000000..af4d856 --- /dev/null +++ b/scripts/install-coz.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Install the coz causal profiler (SPEC §12). Tries the distro package +# first, then a source build. Idempotent — skips if `coz` is already on +# PATH. Run as a user that can apt-install (root in the dev container). +set -eu + +if command -v coz >/dev/null 2>&1; then + echo "coz already installed: $(command -v coz)" + exit 0 +fi + +echo "installing coz…" +if apt-get install -y coz-profiler >/dev/null 2>&1; then + echo "installed via apt: $(command -v coz)" + exit 0 +fi + +# Source build fallback. +apt-get install -y python3 libelf-dev libdwarf-dev build-essential git >/dev/null 2>&1 || true +SRC=/opt/coz-src +rm -rf "$SRC" +git clone --depth 1 https://github.com/plasma-umass/coz.git "$SRC" +make -C "$SRC" >/dev/null +make -C "$SRC" install >/dev/null +command -v coz >/dev/null 2>&1 && echo "installed from source: $(command -v coz)" || { + echo "coz install failed" >&2 + exit 1 +} diff --git a/targets/cjson/harness.c b/targets/cjson/harness.c new file mode 100644 index 0000000..639b12b --- /dev/null +++ b/targets/cjson/harness.c @@ -0,0 +1,73 @@ +/* hotpath bench/golden driver for the cjson target. + * + * TRUST-LAYER CODE — lives outside targets/cjson/workspace, so the agent's + * propose_patch allowlist (workspace-relative paths only) cannot touch it. + * It is compiled together with the (patched) workspace cJSON.c, so a patch + * changes the parser/printer under test but never this measurement rig. + * + * Workload: parse a JSON document `iters` times, serialize it unformatted + * each time, and fold every serialized byte into a rolling checksum so the + * optimizer cannot elide the work. Deterministic: the last serialization + * plus the checksum are written to stdout, giving golden-replay a + * byte-exact signal over both the parse AND the print path. + * + * cjson-bench [iters=40] + */ +#include +#include +#include +#include "cJSON.h" + +/* coz causal profiling (SPEC §12): compile with -DHOTPATH_COZ and link + * libcoz to mark each parse+print iteration as a throughput progress + * point, so `coz run` can measure which source lines, if sped up, would + * actually raise end-to-end throughput. A no-op in normal builds. */ +#ifdef HOTPATH_COZ +#include +#define HOTPATH_PROGRESS() COZ_PROGRESS +#else +#define HOTPATH_PROGRESS() ((void)0) +#endif + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s [iters]\n", argv[0]); + return 2; + } + long iters = (argc >= 3) ? atol(argv[2]) : 40; + if (iters < 1) iters = 1; + + FILE *f = fopen(argv[1], "rb"); + if (!f) { perror("fopen"); return 2; } + fseek(f, 0, SEEK_END); + long n = ftell(f); + fseek(f, 0, SEEK_SET); + if (n < 0) { fclose(f); return 2; } + char *buf = (char *)malloc((size_t)n + 1); + if (!buf) { fclose(f); return 2; } + if (fread(buf, 1, (size_t)n, f) != (size_t)n) { free(buf); fclose(f); return 2; } + buf[n] = '\0'; + fclose(f); + + unsigned long acc = 1469598103934665603UL; /* FNV-ish rolling checksum */ + char *last = NULL; + for (long i = 0; i < iters; i++) { + cJSON *root = cJSON_Parse(buf); + if (!root) { fprintf(stderr, "parse failed\n"); free(buf); return 3; } + char *out = cJSON_PrintUnformatted(root); + if (!out) { cJSON_Delete(root); free(buf); return 3; } + for (char *p = out; *p; p++) acc = (acc ^ (unsigned char)*p) * 1099511628211UL; + if (i == iters - 1) last = out; + else free(out); + cJSON_Delete(root); + HOTPATH_PROGRESS(); /* one parse+print = one throughput unit (coz) */ + } + + if (last) { + fwrite(last, 1, strlen(last), stdout); + free(last); + } + printf("\nchecksum=%lu\n", acc); + free(buf); + return 0; +} diff --git a/targets/cjson/target.toml b/targets/cjson/target.toml new file mode 100644 index 0000000..2d8388a --- /dev/null +++ b/targets/cjson/target.toml @@ -0,0 +1,44 @@ +# cJSON — ultralightweight JSON parser/printer (C) +# Selection vs SPEC §6: CPU-bound parse+serialize; hermetic Unity/CTest +# suite vendored in-tree; deterministic scripted workload; MIT license; +# active but deliberately "simple", no dedicated perf team — the richer +# legacy-C hunting ground Phase 3 calls for. + +[source] +repo = "https://github.com/DaveGamble/cJSON.git" +commit = "c859b25da02955fef659d658b8f324b5cde87be3" # v1.7.19 +license = "MIT" + +[fetch] +# targets/*/workspace is gitignored; reproduce it with: +# git clone https://github.com/DaveGamble/cJSON.git workspace +# git -C workspace checkout v1.7.19 +# The bench/golden driver targets/cjson/harness.c and the Unity suite in +# workspace/tests/ (unity vendored in-tree) are hermetic — no submodules. + +[build] +# Compile the trust-layer harness against the (patched) workspace cJSON.c. +# {out} is the isolation dir; the agent may patch only workspace/*, never +# harness.c, so the measurement rig is fixed across baseline/candidate. +baseline = "mkdir -p {out} && clang -O2 -DNDEBUG -I targets/cjson/workspace -o {out}/cjson-bench targets/cjson/harness.c targets/cjson/workspace/cJSON.c" +binary = "{out}/cjson-bench" +# ASan + UBSan (C/C++ analogue of the Rust nightly ASan path); run on +# every would-be accept (SPEC §8), halting on any leak/UB. +sanitizer = "mkdir -p {out} && clang -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -I targets/cjson/workspace -o {out}/cjson-asan targets/cjson/harness.c targets/cjson/workspace/cJSON.c" +sanitizer_binary = "{out}/cjson-asan" + +[bench] +command = "{binary} corpora/cjson/input/big.json 2 > /dev/null" +workload = "5.6 MB synthetic JSON (20k records: nested objects/arrays, escaped/unicode strings, ints, dense floats), parsed+serialized 2x, single thread" + +[gates] +# Upstream Unity suite via CTest — parse_number/string/array/object, +# print_*, misc_tests, compare/minify. Tests the PATCHED cJSON.c. +tests = "cmake -S targets/cjson/workspace -B targets/cjson/testbuild -DENABLE_CJSON_TEST=On -DCMAKE_BUILD_TYPE=Debug >/dev/null 2>&1 && cmake --build targets/cjson/testbuild -j >/dev/null 2>&1 && ctest --test-dir targets/cjson/testbuild --output-on-failure" +# Golden: full re-serialization + checksum of the corpus, byte-hashed. +golden = "{binary} corpora/cjson/input/big.json 1" + +[corpus] +manifest = "corpora/cjson/MANIFEST.sha256" +root = "corpora/cjson/input" +golden_sha256 = "corpora/cjson/GOLDEN.sha256" diff --git a/targets/comrak/target.toml b/targets/comrak/target.toml index 4fc885d..94b8d67 100644 --- a/targets/comrak/target.toml +++ b/targets/comrak/target.toml @@ -18,8 +18,10 @@ license = "BSD-2-Clause" # git clone workspace/vendor/progit && git -C ... checkout [build] -baseline = "cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml" -binary = "targets/comrak/workspace/target/release/comrak" +baseline = "CARGO_TARGET_DIR={out} cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml" +binary = "{out}/release/comrak" +sanitizer = "CARGO_TARGET_DIR={out} RUSTFLAGS='-Zsanitizer=address' cargo +nightly build --target x86_64-unknown-linux-gnu --manifest-path targets/comrak/workspace/Cargo.toml" +sanitizer_binary = "{out}/x86_64-unknown-linux-gnu/debug/comrak" [bench] # Upstream workload (benches/bench.sh): full Pro Git book through the diff --git a/targets/tokei/target.toml b/targets/tokei/target.toml index f7f33b9..2b84248 100644 --- a/targets/tokei/target.toml +++ b/targets/tokei/target.toml @@ -9,8 +9,10 @@ commit = "fa44e5194060305576514d59b850353643afbfc8" # v14.0.0 license = "MIT OR Apache-2.0" [build] -baseline = "cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml" -binary = "targets/tokei/workspace/target/release/tokei" +baseline = "CARGO_TARGET_DIR={out} cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml" +binary = "{out}/release/tokei" +sanitizer = "CARGO_TARGET_DIR={out} RUSTFLAGS='-Zsanitizer=address' cargo +nightly build --target x86_64-unknown-linux-gnu --manifest-path targets/tokei/workspace/Cargo.toml" +sanitizer_binary = "{out}/x86_64-unknown-linux-gnu/debug/tokei" [bench] # Single rayon thread: measurement first (SPEC conventions); the