mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
phase3: generalize trust layer for C/C++; wire cJSON target
Trust-layer generalization (zero-regression verified on tokei):
- TargetSpec [build] uses an explicit {out} isolation-dir placeholder
instead of injected CARGO_TARGET_DIR, plus optional sanitizer/
sanitizer_binary templates. Nothing assumes cargo anymore.
- verdict rebuild_pristine_baseline, sanitizer_check, and harnessd
run_verdict candidate build all use {out} substitution. A target with
no sanitizer build caps accepts at needs-human-review.
- tokei/comrak target.toml migrated to the {out} form (CARGO_TARGET_DIR=
{out} + nightly ASan sanitizer command).
cJSON C target (v1.7.19, MIT):
- harness.c bench/golden driver (trust-layer; agent can patch workspace
cJSON.c but not the rig), compiled with clang against the patched tree.
- 5.6 MB deterministic corpus (gen-corpus.sh) + MANIFEST/GOLDEN pins.
- gates = hermetic Unity/CTest suite (19 tests pass); sanitizer =
ASan+UBSan; scripts/diff-fuzz-cjson.py for differential fuzzing.
coz causal profiler integration (SPEC §12):
- harness.c optional -DHOTPATH_COZ throughput progress point.
- scripts/install-coz.sh, coz-profile.sh, coz-summary.py; just coz/install-coz.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
Claude
@@ -239,3 +239,7 @@ targets/*/baseline/
|
||||
# Phase 2 agent artifacts (transcripts kept; pending logs are scratch)
|
||||
results/pending/
|
||||
agent/**/__pycache__/
|
||||
|
||||
# cjson corpus is regenerable via gen-corpus.sh (deterministic)
|
||||
corpora/cjson/input/
|
||||
targets/cjson/testbuild/
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
9273ad45a9aa51dd7f941d938a771de3e95a7b3ac807a880f24392c1eb6a72bb golden-stdout
|
||||
@@ -0,0 +1 @@
|
||||
3b0c1dee5a75de59b12e692780896f509c93f3c6b8da7cc337ae4df7fbf70075 ./big.json
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
# Deterministically generate the cjson corpus — a large synthetic JSON
|
||||
# document exercising every parse path: nested objects/arrays, strings
|
||||
# with escapes and unicode, ints, floats (the strtod/sprintf hot path),
|
||||
# booleans, nulls. No randomness — byte-identical on every machine, so
|
||||
# the MANIFEST.sha256 pin is reproducible. Run from repo root:
|
||||
# sh corpora/cjson/gen-corpus.sh
|
||||
set -eu
|
||||
DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
OUT="$DIR/input/big.json"
|
||||
mkdir -p "$DIR/input"
|
||||
|
||||
python3 - "$OUT" <<'PY'
|
||||
import json, sys
|
||||
|
||||
# Deterministic: a fixed linear-congruential stream, no `random` module,
|
||||
# so output is identical everywhere.
|
||||
class Det:
|
||||
def __init__(self, seed=2463534242):
|
||||
self.s = seed
|
||||
def next(self):
|
||||
self.s ^= (self.s << 13) & 0xFFFFFFFF
|
||||
self.s ^= (self.s >> 17)
|
||||
self.s ^= (self.s << 5) & 0xFFFFFFFF
|
||||
return self.s & 0xFFFFFFFF
|
||||
def rng(self, n):
|
||||
return self.next() % n
|
||||
|
||||
d = Det()
|
||||
WORDS = ["alpha","bravo","charlie","delta","echo","foxtrot","golf","hotel",
|
||||
"quote\"inside","tab\tchar","new\nline","back\\slash","unécode","☃snow"]
|
||||
|
||||
def record(i):
|
||||
return {
|
||||
"id": i,
|
||||
"guid": f"{d.next():08x}-{d.next():04x}",
|
||||
"active": bool(d.rng(2)),
|
||||
"score": d.rng(1000000) / 1000.0, # float -> strtod/sprintf path
|
||||
"ratio": d.rng(1 << 30) / (1 << 30), # dense fractional float
|
||||
"count": d.rng(1 << 31) - (1 << 30), # signed int
|
||||
"name": WORDS[d.rng(len(WORDS))] + str(d.rng(9999)),
|
||||
"tags": [WORDS[d.rng(len(WORDS))] for _ in range(d.rng(6))],
|
||||
"coords": [d.rng(1 << 20) / 997.0 for _ in range(3)],
|
||||
"meta": None if d.rng(4) == 0 else {
|
||||
"created": d.next(),
|
||||
"nested": {"a": d.rng(100), "b": [d.rng(50) for _ in range(d.rng(4))]},
|
||||
},
|
||||
}
|
||||
|
||||
doc = {
|
||||
"schema": "hotpath.cjson.corpus/v1",
|
||||
"generated_by": "gen-corpus.sh (deterministic)",
|
||||
"records": [record(i) for i in range(20000)],
|
||||
}
|
||||
with open(sys.argv[1], "w") as f:
|
||||
json.dump(doc, f, ensure_ascii=False, separators=(",", ":"))
|
||||
PY
|
||||
|
||||
echo "wrote $OUT ($(wc -c < "$OUT") bytes)"
|
||||
# Pin the corpus and regenerate the manifest.
|
||||
( cd "$DIR/input" && find . -type f -print0 | sort -z | xargs -0 sha256sum ) > "$DIR/MANIFEST.sha256"
|
||||
echo "manifest: $DIR/MANIFEST.sha256"
|
||||
@@ -33,10 +33,31 @@ pub struct Source {
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Build {
|
||||
/// Baseline build command, run from repo root.
|
||||
/// Build command, run from repo root. `{out}` is substituted with an
|
||||
/// isolation directory the harness provides (per-baseline, per-
|
||||
/// candidate, per-sanitizer) — the command MUST place all artifacts
|
||||
/// under it. A cargo target sets `CARGO_TARGET_DIR={out}`; a make
|
||||
/// target builds and copies the binary to `{out}`. Language-agnostic:
|
||||
/// nothing here assumes cargo.
|
||||
pub baseline: String,
|
||||
/// Built binary path, relative to repo root.
|
||||
/// Built binary path with `{out}` substituted — where `baseline`
|
||||
/// leaves the binary given an isolation dir.
|
||||
pub binary: String,
|
||||
/// ASan+LSan (or ASan+UBSan for C/C++) build command; `{out}`
|
||||
/// substituted. Run on every would-be accept (SPEC §8). If absent,
|
||||
/// an accept cannot be sanitizer-verified and is capped at
|
||||
/// needs-human-review.
|
||||
#[serde(default)]
|
||||
pub sanitizer: Option<String>,
|
||||
/// Sanitizer binary path with `{out}` substituted.
|
||||
#[serde(default)]
|
||||
pub sanitizer_binary: Option<String>,
|
||||
}
|
||||
|
||||
/// Substitute the `{out}` isolation-dir placeholder in a build command or
|
||||
/// binary-path template.
|
||||
pub fn subst_out(template: &str, out_dir: &str) -> String {
|
||||
template.replace("{out}", out_dir)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
|
||||
+11
-10
@@ -339,29 +339,30 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
|
||||
)?)?;
|
||||
let t = pending["target"].as_str().unwrap();
|
||||
let spec = diff_test::target::TargetSpec::load(root, t)?;
|
||||
let bin = Path::new(&spec.build.binary)
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
||||
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
|
||||
// The build + verdict pipeline runs for minutes — far past the MCP
|
||||
// transport's per-call cap. Launch it detached, writing progress to
|
||||
// a log; the agent observes the result via the read_verdict op,
|
||||
// which reads the append-only ledger row once written.
|
||||
let cand_dir = format!("targets/{t}/candidate-{patch_id}");
|
||||
let cand_bin = format!("{cand_dir}/release/{bin}");
|
||||
// {out} isolates the candidate build (cargo: CARGO_TARGET_DIR;
|
||||
// make: build+copy into it) — language-agnostic, mirrors the
|
||||
// baseline rebuild in verdict.
|
||||
let cand_dir = root
|
||||
.join(format!("targets/{t}/candidate-{patch_id}"))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let cand_build = diff_test::target::subst_out(&spec.build.baseline, &cand_dir);
|
||||
let cand_bin = diff_test::target::subst_out(&spec.build.binary, &cand_dir);
|
||||
let log = format!("results/pending/{run_id}.log");
|
||||
// On any failure (candidate build, gates crash) the trap line
|
||||
// marks the log so read_verdict can report "failed" instead of
|
||||
// leaving the agent polling a verdict that will never exist.
|
||||
let script = format!(
|
||||
"{{ set -e\nCARGO_TARGET_DIR={cand} {build}\ncargo run -q -p verdict -- {tgt} \
|
||||
"{{ set -e\n{build}\ncargo run -q -p verdict -- {tgt} \
|
||||
--rebuild-baseline --candidate-bin {cbin} --run-id {rid} --playbook-class {cls} \
|
||||
--hypothesis {hyp} --hotspot {hs} --patch-file {pf}\n}} || echo HOTPATH-PIPELINE-FAILED\n",
|
||||
cand = shq(&cand_dir),
|
||||
build = spec.build.baseline,
|
||||
build = cand_build,
|
||||
tgt = shq(t),
|
||||
cbin = shq(&cand_bin),
|
||||
rid = shq(run_id),
|
||||
|
||||
+48
-42
@@ -79,14 +79,14 @@ fn rebuild_pristine_baseline(
|
||||
"git stash failed"
|
||||
);
|
||||
}
|
||||
let out_dir = root
|
||||
.join(format!("targets/{target}/baseline"))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let build = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(&spec.build.baseline)
|
||||
.arg(diff_test::target::subst_out(&spec.build.baseline, &out_dir))
|
||||
.current_dir(root)
|
||||
.env(
|
||||
"CARGO_TARGET_DIR",
|
||||
root.join(format!("targets/{target}/baseline")),
|
||||
)
|
||||
.status();
|
||||
if dirty {
|
||||
anyhow::ensure!(
|
||||
@@ -98,13 +98,9 @@ fn rebuild_pristine_baseline(
|
||||
);
|
||||
}
|
||||
anyhow::ensure!(build?.success(), "baseline build failed");
|
||||
let bin_name = std::path::Path::new(&spec.build.binary)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
||||
let baseline = format!("targets/{target}/baseline/release/{bin_name}");
|
||||
let baseline = diff_test::target::subst_out(&spec.build.binary, &out_dir);
|
||||
anyhow::ensure!(
|
||||
root.join(&baseline).exists(),
|
||||
std::path::Path::new(&baseline).exists(),
|
||||
"baseline binary missing: {baseline}"
|
||||
);
|
||||
println!("baseline rebuilt from pristine checkout: {baseline}");
|
||||
@@ -117,40 +113,43 @@ fn rebuild_pristine_baseline(
|
||||
/// were per-attempt manual) and the human audit had to overturn it — the
|
||||
/// accept path now runs the check itself and caps flagged wins at
|
||||
/// needs-human-review. Rejections skip it: they ship nothing.
|
||||
fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result<bool> {
|
||||
let ws = root.join(format!("targets/{target}/workspace"));
|
||||
let asan_dir = root.join(format!("targets/{target}/asan"));
|
||||
println!("sanitizers: ASan+LSan build (nightly) of the patched tree…");
|
||||
let status = std::process::Command::new("cargo")
|
||||
.args([
|
||||
"+nightly",
|
||||
"build",
|
||||
"-q",
|
||||
"--target",
|
||||
"x86_64-unknown-linux-gnu",
|
||||
])
|
||||
.env("RUSTFLAGS", "-Zsanitizer=address")
|
||||
.env("CARGO_TARGET_DIR", &asan_dir)
|
||||
.current_dir(&ws)
|
||||
/// Returns Some(clean) if the sanitizer gate ran, or None if the target
|
||||
/// declares no sanitizer build (an accept then cannot be verified and is
|
||||
/// capped at needs-human-review by the caller).
|
||||
fn sanitizer_check(
|
||||
root: &std::path::Path,
|
||||
target: &str,
|
||||
spec: &TargetSpec,
|
||||
) -> Result<Option<bool>> {
|
||||
let (Some(build_tpl), Some(bin_tpl)) = (&spec.build.sanitizer, &spec.build.sanitizer_binary)
|
||||
else {
|
||||
println!("sanitizers: target declares no sanitizer build — cannot verify an accept");
|
||||
return Ok(None);
|
||||
};
|
||||
let out_dir = root
|
||||
.join(format!("targets/{target}/asan"))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
println!("sanitizers: ASan build of the patched tree…");
|
||||
let status = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(diff_test::target::subst_out(build_tpl, &out_dir))
|
||||
.current_dir(root)
|
||||
.status()?;
|
||||
anyhow::ensure!(status.success(), "sanitizer build failed");
|
||||
let bin_name = std::path::Path::new(&spec.build.binary)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
||||
let asan_bin = asan_dir.join(format!("x86_64-unknown-linux-gnu/debug/{bin_name}"));
|
||||
let cmd = spec
|
||||
.bench
|
||||
.command
|
||||
.replace("{binary}", asan_bin.to_str().unwrap());
|
||||
let asan_bin = diff_test::target::subst_out(bin_tpl, &out_dir);
|
||||
let cmd = spec.bench.command.replace("{binary}", &asan_bin);
|
||||
let out = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(&cmd)
|
||||
// detect_leaks is a no-op for the C/C++ UBSan combo but harmless.
|
||||
.env("ASAN_OPTIONS", "detect_leaks=1")
|
||||
.env("UBSAN_OPTIONS", "halt_on_error=1:print_stacktrace=1")
|
||||
.current_dir(root)
|
||||
.output()?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let clean = out.status.success() && !stderr.contains("ERROR:");
|
||||
let clean =
|
||||
out.status.success() && !stderr.contains("ERROR:") && !stderr.contains("runtime error:");
|
||||
if clean {
|
||||
println!("sanitizers: clean");
|
||||
} else {
|
||||
@@ -161,7 +160,7 @@ fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> R
|
||||
println!(" {l}");
|
||||
}
|
||||
}
|
||||
Ok(clean)
|
||||
Ok(Some(clean))
|
||||
}
|
||||
|
||||
fn now_utc() -> String {
|
||||
@@ -265,11 +264,18 @@ fn main() -> Result<()> {
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
if v == Verdict::Accepted {
|
||||
let clean = sanitizer_check(&root, &cli.target, &spec)?;
|
||||
gate_results.sanitizers_clean = clean;
|
||||
if !clean {
|
||||
println!("verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)");
|
||||
v = Verdict::NeedsHumanReview;
|
||||
match sanitizer_check(&root, &cli.target, &spec)? {
|
||||
Some(true) => gate_results.sanitizers_clean = true,
|
||||
Some(false) => {
|
||||
println!(
|
||||
"verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)"
|
||||
);
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
None => {
|
||||
println!("verdict: no sanitizer build declared; capping accept at needs-human-review (SPEC §8)");
|
||||
v = Verdict::NeedsHumanReview;
|
||||
}
|
||||
}
|
||||
}
|
||||
let fp = EnvFingerprint::collect(pin, "system-default");
|
||||
|
||||
@@ -56,3 +56,12 @@ agent-attempt target run_id max_turns="40":
|
||||
# Verify the OS boundary from the agent user's side (negative + positive)
|
||||
isolation-check:
|
||||
sh scripts/isolation-check.sh
|
||||
|
||||
# Install the coz causal profiler (apt, else source build)
|
||||
install-coz:
|
||||
sh scripts/install-coz.sh
|
||||
|
||||
# Causal profile of a C/C++ target: where a speedup would raise throughput
|
||||
coz target iters="200":
|
||||
sh scripts/coz-profile.sh {{target}} {{iters}}
|
||||
python3 scripts/coz-summary.py results/{{target}}/coz/profile.coz
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,41 @@
|
||||
#!/bin/sh
|
||||
# coz causal profiling for a C/C++ target (SPEC §12). Unlike a flat
|
||||
# profiler (callgrind ranks where time is SPENT), coz measures where a
|
||||
# speedup would actually RAISE throughput — it virtually speeds up each
|
||||
# line and reports the causal effect on the harness's progress points.
|
||||
# Use it to rank playbook targets when the flat profile is ambiguous.
|
||||
#
|
||||
# scripts/coz-profile.sh cjson [iters]
|
||||
#
|
||||
# Requires: coz on PATH (build: git clone github.com/plasma-umass/coz &&
|
||||
# see scripts/install-coz.sh) and the target's harness built with
|
||||
# -DHOTPATH_COZ (this script does that). Output: results/<target>/coz/
|
||||
set -eu
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
TARGET=${1:?usage: coz-profile.sh <target> [iters]}
|
||||
ITERS=${2:-200}
|
||||
OUT="$ROOT/results/$TARGET/coz"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
command -v coz >/dev/null 2>&1 || {
|
||||
echo "coz not on PATH — run scripts/install-coz.sh first" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# coz needs debug line info and the progress point compiled in.
|
||||
BIN="$OUT/cjson-bench-coz"
|
||||
clang -O2 -g -gdwarf-4 -DHOTPATH_COZ \
|
||||
-I "$ROOT/targets/$TARGET/workspace" \
|
||||
-I "$(dirname "$(command -v coz)")/../include" \
|
||||
-o "$BIN" \
|
||||
"$ROOT/targets/$TARGET/harness.c" \
|
||||
"$ROOT/targets/$TARGET/workspace/cJSON.c" \
|
||||
-ldl
|
||||
|
||||
cd "$ROOT"
|
||||
coz run --output "$OUT/profile.coz" --- \
|
||||
"$BIN" corpora/$TARGET/input/big.json "$ITERS" >/dev/null
|
||||
|
||||
echo "coz profile written: $OUT/profile.coz"
|
||||
echo "view at https://plasma-umass.org/coz/ (load the .coz file) or:"
|
||||
echo " python3 scripts/coz-summary.py $OUT/profile.coz"
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize a coz causal profile: rank source lines by their estimated
|
||||
causal effect on throughput. coz writes newline-delimited JSON records;
|
||||
for each (line, virtual-speedup) it records progress-point throughput.
|
||||
We fit the slope of throughput-gain vs virtual-speedup per line — a
|
||||
steeper positive slope means "speeding up this line most raises overall
|
||||
throughput", which is exactly the class-selection signal a flat profiler
|
||||
can't give.
|
||||
|
||||
python3 scripts/coz-summary.py results/<target>/coz/profile.coz
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = Path(sys.argv[1]) if len(sys.argv) > 1 else None
|
||||
if not path or not path.exists():
|
||||
print("usage: coz-summary.py <profile.coz>", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# experiments: line -> list of (speedup, progress-rate)
|
||||
exps: dict[str, list[tuple[float, float]]] = collections.defaultdict(list)
|
||||
baseline: dict[str, float] = {}
|
||||
for raw in path.read_text().splitlines():
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if rec.get("type") != "experiment":
|
||||
continue
|
||||
line = rec.get("selected", "?")
|
||||
speedup = float(rec.get("speedup", 0.0))
|
||||
duration = float(rec.get("duration", 0.0))
|
||||
delta = float(rec.get("delta", 0.0))
|
||||
if delta <= 0:
|
||||
continue
|
||||
rate = duration / delta # time per progress unit; lower = faster
|
||||
exps[line].append((speedup, rate))
|
||||
if speedup == 0.0:
|
||||
baseline[line] = rate
|
||||
|
||||
rows = []
|
||||
for line, pts in exps.items():
|
||||
base = baseline.get(line)
|
||||
if not base or len(pts) < 2:
|
||||
continue
|
||||
# % throughput improvement at the max virtual speedup tried
|
||||
pts.sort()
|
||||
s_max, r_max = pts[-1]
|
||||
if s_max <= 0 or r_max <= 0:
|
||||
continue
|
||||
gain = (base - r_max) / base # fractional throughput gain
|
||||
rows.append((gain, s_max, line))
|
||||
|
||||
rows.sort(reverse=True)
|
||||
print(f"{'throughput_gain':>16} {'@speedup':>9} line")
|
||||
print("-" * 70)
|
||||
for gain, s_max, line in rows[:25]:
|
||||
print(f"{gain * 100:>14.1f}% {s_max * 100:>7.0f}% {line}")
|
||||
if not rows:
|
||||
print("(no usable experiments — did the run mark progress points?)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Differential fuzz for cjson attempts: pristine vs candidate harness must
|
||||
produce byte-identical stdout (serialization + checksum) AND identical exit
|
||||
code on mutated JSON. cJSON is a deterministic single-threaded parser, so
|
||||
raw byte comparison needs no canonicalization.
|
||||
|
||||
python3 scripts/diff-fuzz-cjson.py <workdir> <iters> <candidate-bin>
|
||||
|
||||
Seeds: slices of the pinned corpus plus hand JSON fragments. Mutations
|
||||
target parser edges: structural tokens, string escapes, number forms
|
||||
(leading zeros, +/- exponents, huge/deep values), and raw byte noise.
|
||||
"""
|
||||
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path("/home/user/peltier.io")
|
||||
BASE = ROOT / "targets/cjson/baseline/cjson-bench"
|
||||
CAND = ROOT / sys.argv[3] if len(sys.argv) > 3 else ROOT / "targets/cjson/baseline/cjson-bench"
|
||||
CORPUS = ROOT / "corpora/cjson/input/big.json"
|
||||
WORK = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/tmp/difffuzz-cjson")
|
||||
ITERS = int(sys.argv[2]) if len(sys.argv) > 2 else 10000
|
||||
|
||||
TOKENS = [
|
||||
b"{", b"}", b"[", b"]", b'"', b":", b",", b"\\", b"\\n", b"\\u00e9", b"\\uD83D",
|
||||
b"true", b"false", b"null", b"0", b"-0", b"1e999", b"-1.5E-3", b"00", b"1.",
|
||||
b".5", b"1e", b"NaN", b"Infinity", b'\\"', b'"key"', b"[]", b"{}", b'""',
|
||||
b"1234567890123456789012345", b"\x00", b"\xff", b"\t", b"\n",
|
||||
]
|
||||
|
||||
SEEDS_STATIC = [
|
||||
b'{}', b'[]', b'{"a":1}', b'[1,2,3]', b'"hello"', b'true', b'null',
|
||||
b'{"n":-1.5e10,"s":"a\\tb","u":"\\u00e9","arr":[null,false,{}]}',
|
||||
b'[[[[[[1]]]]]]', b'{"a":{"b":{"c":{"d":1}}}}', b'123.456e-78',
|
||||
b'{"esc":"\\"\\\\\\/\\b\\f\\n\\r\\t"}', b' { "x" : [ 1 , 2 ] } ',
|
||||
]
|
||||
|
||||
|
||||
def mutate(data: bytes, rng: random.Random) -> bytes:
|
||||
b = bytearray(data)
|
||||
for _ in range(rng.randrange(1, 8)):
|
||||
if not b:
|
||||
b = bytearray(rng.choice(TOKENS))
|
||||
continue
|
||||
op = rng.randrange(5)
|
||||
pos = rng.randrange(len(b) + 1)
|
||||
if op == 0:
|
||||
b[pos:pos] = rng.choice(TOKENS)
|
||||
elif op == 1 and len(b) > 2:
|
||||
del b[pos:min(len(b), pos + rng.randrange(1, 12))]
|
||||
elif op == 2 and pos < len(b):
|
||||
b[pos] = rng.randrange(256)
|
||||
elif op == 3:
|
||||
end = min(len(b), pos + rng.randrange(1, 48))
|
||||
b[end:end] = b[pos:end]
|
||||
else:
|
||||
b[pos:pos] = rng.choice(TOKENS)
|
||||
return bytes(b)
|
||||
|
||||
|
||||
def run(binary: Path, path: Path) -> tuple[bytes, int]:
|
||||
r = subprocess.run([str(binary), str(path), "1"], capture_output=True, timeout=30)
|
||||
return r.stdout, r.returncode
|
||||
|
||||
|
||||
def main() -> int:
|
||||
rng = random.Random(0x1CE_B00C)
|
||||
corpus = CORPUS.read_bytes()
|
||||
seeds = list(SEEDS_STATIC)
|
||||
# windows of the real corpus (often invalid mid-slice — exercises the
|
||||
# error path, which must diverge identically or not at all)
|
||||
for o in range(0, len(corpus) - 4096, len(corpus) // 200):
|
||||
for n in (64, 512, 4096):
|
||||
seeds.append(corpus[o : o + n])
|
||||
WORK.mkdir(parents=True, exist_ok=True)
|
||||
case = WORK / "case.json"
|
||||
divergences = 0
|
||||
for i in range(ITERS):
|
||||
payload = mutate(rng.choice(seeds), rng)
|
||||
case.write_bytes(payload)
|
||||
a_out, a_rc = run(BASE, case)
|
||||
b_out, b_rc = run(CAND, case)
|
||||
if a_out != b_out or a_rc != b_rc:
|
||||
divergences += 1
|
||||
keep = WORK / f"divergence-{i}"
|
||||
keep.mkdir(exist_ok=True)
|
||||
(keep / "input.json").write_bytes(payload)
|
||||
(keep / "baseline.out").write_bytes(a_out + f"\nrc={a_rc}".encode())
|
||||
(keep / "candidate.out").write_bytes(b_out + f"\nrc={b_rc}".encode())
|
||||
print(f"DIVERGENCE at iter {i} (rc {a_rc} vs {b_rc}); kept in {keep}")
|
||||
if (i + 1) % 1000 == 0:
|
||||
print(f"{i + 1}/{ITERS}, {divergences} divergences", flush=True)
|
||||
print(f"done: {ITERS} inputs, {divergences} divergences")
|
||||
return 1 if divergences else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# Install the coz causal profiler (SPEC §12). Tries the distro package
|
||||
# first, then a source build. Idempotent — skips if `coz` is already on
|
||||
# PATH. Run as a user that can apt-install (root in the dev container).
|
||||
set -eu
|
||||
|
||||
if command -v coz >/dev/null 2>&1; then
|
||||
echo "coz already installed: $(command -v coz)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "installing coz…"
|
||||
if apt-get install -y coz-profiler >/dev/null 2>&1; then
|
||||
echo "installed via apt: $(command -v coz)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Source build fallback.
|
||||
apt-get install -y python3 libelf-dev libdwarf-dev build-essential git >/dev/null 2>&1 || true
|
||||
SRC=/opt/coz-src
|
||||
rm -rf "$SRC"
|
||||
git clone --depth 1 https://github.com/plasma-umass/coz.git "$SRC"
|
||||
make -C "$SRC" >/dev/null
|
||||
make -C "$SRC" install >/dev/null
|
||||
command -v coz >/dev/null 2>&1 && echo "installed from source: $(command -v coz)" || {
|
||||
echo "coz install failed" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/* hotpath bench/golden driver for the cjson target.
|
||||
*
|
||||
* TRUST-LAYER CODE — lives outside targets/cjson/workspace, so the agent's
|
||||
* propose_patch allowlist (workspace-relative paths only) cannot touch it.
|
||||
* It is compiled together with the (patched) workspace cJSON.c, so a patch
|
||||
* changes the parser/printer under test but never this measurement rig.
|
||||
*
|
||||
* Workload: parse a JSON document `iters` times, serialize it unformatted
|
||||
* each time, and fold every serialized byte into a rolling checksum so the
|
||||
* optimizer cannot elide the work. Deterministic: the last serialization
|
||||
* plus the checksum are written to stdout, giving golden-replay a
|
||||
* byte-exact signal over both the parse AND the print path.
|
||||
*
|
||||
* cjson-bench <file.json> [iters=40]
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "cJSON.h"
|
||||
|
||||
/* coz causal profiling (SPEC §12): compile with -DHOTPATH_COZ and link
|
||||
* libcoz to mark each parse+print iteration as a throughput progress
|
||||
* point, so `coz run` can measure which source lines, if sped up, would
|
||||
* actually raise end-to-end throughput. A no-op in normal builds. */
|
||||
#ifdef HOTPATH_COZ
|
||||
#include <coz.h>
|
||||
#define HOTPATH_PROGRESS() COZ_PROGRESS
|
||||
#else
|
||||
#define HOTPATH_PROGRESS() ((void)0)
|
||||
#endif
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "usage: %s <file.json> [iters]\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
long iters = (argc >= 3) ? atol(argv[2]) : 40;
|
||||
if (iters < 1) iters = 1;
|
||||
|
||||
FILE *f = fopen(argv[1], "rb");
|
||||
if (!f) { perror("fopen"); return 2; }
|
||||
fseek(f, 0, SEEK_END);
|
||||
long n = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
if (n < 0) { fclose(f); return 2; }
|
||||
char *buf = (char *)malloc((size_t)n + 1);
|
||||
if (!buf) { fclose(f); return 2; }
|
||||
if (fread(buf, 1, (size_t)n, f) != (size_t)n) { free(buf); fclose(f); return 2; }
|
||||
buf[n] = '\0';
|
||||
fclose(f);
|
||||
|
||||
unsigned long acc = 1469598103934665603UL; /* FNV-ish rolling checksum */
|
||||
char *last = NULL;
|
||||
for (long i = 0; i < iters; i++) {
|
||||
cJSON *root = cJSON_Parse(buf);
|
||||
if (!root) { fprintf(stderr, "parse failed\n"); free(buf); return 3; }
|
||||
char *out = cJSON_PrintUnformatted(root);
|
||||
if (!out) { cJSON_Delete(root); free(buf); return 3; }
|
||||
for (char *p = out; *p; p++) acc = (acc ^ (unsigned char)*p) * 1099511628211UL;
|
||||
if (i == iters - 1) last = out;
|
||||
else free(out);
|
||||
cJSON_Delete(root);
|
||||
HOTPATH_PROGRESS(); /* one parse+print = one throughput unit (coz) */
|
||||
}
|
||||
|
||||
if (last) {
|
||||
fwrite(last, 1, strlen(last), stdout);
|
||||
free(last);
|
||||
}
|
||||
printf("\nchecksum=%lu\n", acc);
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# cJSON — ultralightweight JSON parser/printer (C)
|
||||
# Selection vs SPEC §6: CPU-bound parse+serialize; hermetic Unity/CTest
|
||||
# suite vendored in-tree; deterministic scripted workload; MIT license;
|
||||
# active but deliberately "simple", no dedicated perf team — the richer
|
||||
# legacy-C hunting ground Phase 3 calls for.
|
||||
|
||||
[source]
|
||||
repo = "https://github.com/DaveGamble/cJSON.git"
|
||||
commit = "c859b25da02955fef659d658b8f324b5cde87be3" # v1.7.19
|
||||
license = "MIT"
|
||||
|
||||
[fetch]
|
||||
# targets/*/workspace is gitignored; reproduce it with:
|
||||
# git clone https://github.com/DaveGamble/cJSON.git workspace
|
||||
# git -C workspace checkout v1.7.19
|
||||
# The bench/golden driver targets/cjson/harness.c and the Unity suite in
|
||||
# workspace/tests/ (unity vendored in-tree) are hermetic — no submodules.
|
||||
|
||||
[build]
|
||||
# Compile the trust-layer harness against the (patched) workspace cJSON.c.
|
||||
# {out} is the isolation dir; the agent may patch only workspace/*, never
|
||||
# harness.c, so the measurement rig is fixed across baseline/candidate.
|
||||
baseline = "mkdir -p {out} && clang -O2 -DNDEBUG -I targets/cjson/workspace -o {out}/cjson-bench targets/cjson/harness.c targets/cjson/workspace/cJSON.c"
|
||||
binary = "{out}/cjson-bench"
|
||||
# ASan + UBSan (C/C++ analogue of the Rust nightly ASan path); run on
|
||||
# every would-be accept (SPEC §8), halting on any leak/UB.
|
||||
sanitizer = "mkdir -p {out} && clang -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -I targets/cjson/workspace -o {out}/cjson-asan targets/cjson/harness.c targets/cjson/workspace/cJSON.c"
|
||||
sanitizer_binary = "{out}/cjson-asan"
|
||||
|
||||
[bench]
|
||||
command = "{binary} corpora/cjson/input/big.json 2 > /dev/null"
|
||||
workload = "5.6 MB synthetic JSON (20k records: nested objects/arrays, escaped/unicode strings, ints, dense floats), parsed+serialized 2x, single thread"
|
||||
|
||||
[gates]
|
||||
# Upstream Unity suite via CTest — parse_number/string/array/object,
|
||||
# print_*, misc_tests, compare/minify. Tests the PATCHED cJSON.c.
|
||||
tests = "cmake -S targets/cjson/workspace -B targets/cjson/testbuild -DENABLE_CJSON_TEST=On -DCMAKE_BUILD_TYPE=Debug >/dev/null 2>&1 && cmake --build targets/cjson/testbuild -j >/dev/null 2>&1 && ctest --test-dir targets/cjson/testbuild --output-on-failure"
|
||||
# Golden: full re-serialization + checksum of the corpus, byte-hashed.
|
||||
golden = "{binary} corpora/cjson/input/big.json 1"
|
||||
|
||||
[corpus]
|
||||
manifest = "corpora/cjson/MANIFEST.sha256"
|
||||
root = "corpora/cjson/input"
|
||||
golden_sha256 = "corpora/cjson/GOLDEN.sha256"
|
||||
@@ -18,8 +18,10 @@ license = "BSD-2-Clause"
|
||||
# git clone <submodule repo> workspace/vendor/progit && git -C ... checkout <commit>
|
||||
|
||||
[build]
|
||||
baseline = "cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml"
|
||||
binary = "targets/comrak/workspace/target/release/comrak"
|
||||
baseline = "CARGO_TARGET_DIR={out} cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml"
|
||||
binary = "{out}/release/comrak"
|
||||
sanitizer = "CARGO_TARGET_DIR={out} RUSTFLAGS='-Zsanitizer=address' cargo +nightly build --target x86_64-unknown-linux-gnu --manifest-path targets/comrak/workspace/Cargo.toml"
|
||||
sanitizer_binary = "{out}/x86_64-unknown-linux-gnu/debug/comrak"
|
||||
|
||||
[bench]
|
||||
# Upstream workload (benches/bench.sh): full Pro Git book through the
|
||||
|
||||
@@ -9,8 +9,10 @@ commit = "fa44e5194060305576514d59b850353643afbfc8" # v14.0.0
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
[build]
|
||||
baseline = "cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml"
|
||||
binary = "targets/tokei/workspace/target/release/tokei"
|
||||
baseline = "CARGO_TARGET_DIR={out} cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml"
|
||||
binary = "{out}/release/tokei"
|
||||
sanitizer = "CARGO_TARGET_DIR={out} RUSTFLAGS='-Zsanitizer=address' cargo +nightly build --target x86_64-unknown-linux-gnu --manifest-path targets/tokei/workspace/Cargo.toml"
|
||||
sanitizer_binary = "{out}/x86_64-unknown-linux-gnu/debug/tokei"
|
||||
|
||||
[bench]
|
||||
# Single rayon thread: measurement first (SPEC conventions); the
|
||||
|
||||
Reference in New Issue
Block a user