mirror of
https://github.com/bdbrown4/peltier.io.git
synced 2026-11-07 10:58:28 +00:00
phase3: generalize trust layer for C/C++; wire cJSON target
Trust-layer generalization (zero-regression verified on tokei):
- TargetSpec [build] uses an explicit {out} isolation-dir placeholder
instead of injected CARGO_TARGET_DIR, plus optional sanitizer/
sanitizer_binary templates. Nothing assumes cargo anymore.
- verdict rebuild_pristine_baseline, sanitizer_check, and harnessd
run_verdict candidate build all use {out} substitution. A target with
no sanitizer build caps accepts at needs-human-review.
- tokei/comrak target.toml migrated to the {out} form (CARGO_TARGET_DIR=
{out} + nightly ASan sanitizer command).
cJSON C target (v1.7.19, MIT):
- harness.c bench/golden driver (trust-layer; agent can patch workspace
cJSON.c but not the rig), compiled with clang against the patched tree.
- 5.6 MB deterministic corpus (gen-corpus.sh) + MANIFEST/GOLDEN pins.
- gates = hermetic Unity/CTest suite (19 tests pass); sanitizer =
ASan+UBSan; scripts/diff-fuzz-cjson.py for differential fuzzing.
coz causal profiler integration (SPEC §12):
- harness.c optional -DHOTPATH_COZ throughput progress point.
- scripts/install-coz.sh, coz-profile.sh, coz-summary.py; just coz/install-coz.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EcKPWQ3MQD4NdEHwar96u2
This commit is contained in:
@@ -239,3 +239,7 @@ targets/*/baseline/
|
|||||||
# Phase 2 agent artifacts (transcripts kept; pending logs are scratch)
|
# Phase 2 agent artifacts (transcripts kept; pending logs are scratch)
|
||||||
results/pending/
|
results/pending/
|
||||||
agent/**/__pycache__/
|
agent/**/__pycache__/
|
||||||
|
|
||||||
|
# cjson corpus is regenerable via gen-corpus.sh (deterministic)
|
||||||
|
corpora/cjson/input/
|
||||||
|
targets/cjson/testbuild/
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
9273ad45a9aa51dd7f941d938a771de3e95a7b3ac807a880f24392c1eb6a72bb golden-stdout
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
3b0c1dee5a75de59b12e692780896f509c93f3c6b8da7cc337ae4df7fbf70075 ./big.json
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Deterministically generate the cjson corpus — a large synthetic JSON
|
||||||
|
# document exercising every parse path: nested objects/arrays, strings
|
||||||
|
# with escapes and unicode, ints, floats (the strtod/sprintf hot path),
|
||||||
|
# booleans, nulls. No randomness — byte-identical on every machine, so
|
||||||
|
# the MANIFEST.sha256 pin is reproducible. Run from repo root:
|
||||||
|
# sh corpora/cjson/gen-corpus.sh
|
||||||
|
set -eu
|
||||||
|
DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
OUT="$DIR/input/big.json"
|
||||||
|
mkdir -p "$DIR/input"
|
||||||
|
|
||||||
|
python3 - "$OUT" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
|
||||||
|
# Deterministic: a fixed linear-congruential stream, no `random` module,
|
||||||
|
# so output is identical everywhere.
|
||||||
|
class Det:
|
||||||
|
def __init__(self, seed=2463534242):
|
||||||
|
self.s = seed
|
||||||
|
def next(self):
|
||||||
|
self.s ^= (self.s << 13) & 0xFFFFFFFF
|
||||||
|
self.s ^= (self.s >> 17)
|
||||||
|
self.s ^= (self.s << 5) & 0xFFFFFFFF
|
||||||
|
return self.s & 0xFFFFFFFF
|
||||||
|
def rng(self, n):
|
||||||
|
return self.next() % n
|
||||||
|
|
||||||
|
d = Det()
|
||||||
|
WORDS = ["alpha","bravo","charlie","delta","echo","foxtrot","golf","hotel",
|
||||||
|
"quote\"inside","tab\tchar","new\nline","back\\slash","unécode","☃snow"]
|
||||||
|
|
||||||
|
def record(i):
|
||||||
|
return {
|
||||||
|
"id": i,
|
||||||
|
"guid": f"{d.next():08x}-{d.next():04x}",
|
||||||
|
"active": bool(d.rng(2)),
|
||||||
|
"score": d.rng(1000000) / 1000.0, # float -> strtod/sprintf path
|
||||||
|
"ratio": d.rng(1 << 30) / (1 << 30), # dense fractional float
|
||||||
|
"count": d.rng(1 << 31) - (1 << 30), # signed int
|
||||||
|
"name": WORDS[d.rng(len(WORDS))] + str(d.rng(9999)),
|
||||||
|
"tags": [WORDS[d.rng(len(WORDS))] for _ in range(d.rng(6))],
|
||||||
|
"coords": [d.rng(1 << 20) / 997.0 for _ in range(3)],
|
||||||
|
"meta": None if d.rng(4) == 0 else {
|
||||||
|
"created": d.next(),
|
||||||
|
"nested": {"a": d.rng(100), "b": [d.rng(50) for _ in range(d.rng(4))]},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
doc = {
|
||||||
|
"schema": "hotpath.cjson.corpus/v1",
|
||||||
|
"generated_by": "gen-corpus.sh (deterministic)",
|
||||||
|
"records": [record(i) for i in range(20000)],
|
||||||
|
}
|
||||||
|
with open(sys.argv[1], "w") as f:
|
||||||
|
json.dump(doc, f, ensure_ascii=False, separators=(",", ":"))
|
||||||
|
PY
|
||||||
|
|
||||||
|
echo "wrote $OUT ($(wc -c < "$OUT") bytes)"
|
||||||
|
# Pin the corpus and regenerate the manifest.
|
||||||
|
( cd "$DIR/input" && find . -type f -print0 | sort -z | xargs -0 sha256sum ) > "$DIR/MANIFEST.sha256"
|
||||||
|
echo "manifest: $DIR/MANIFEST.sha256"
|
||||||
@@ -33,10 +33,31 @@ pub struct Source {
|
|||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub struct Build {
|
pub struct Build {
|
||||||
/// Baseline build command, run from repo root.
|
/// Build command, run from repo root. `{out}` is substituted with an
|
||||||
|
/// isolation directory the harness provides (per-baseline, per-
|
||||||
|
/// candidate, per-sanitizer) — the command MUST place all artifacts
|
||||||
|
/// under it. A cargo target sets `CARGO_TARGET_DIR={out}`; a make
|
||||||
|
/// target builds and copies the binary to `{out}`. Language-agnostic:
|
||||||
|
/// nothing here assumes cargo.
|
||||||
pub baseline: String,
|
pub baseline: String,
|
||||||
/// Built binary path, relative to repo root.
|
/// Built binary path with `{out}` substituted — where `baseline`
|
||||||
|
/// leaves the binary given an isolation dir.
|
||||||
pub binary: String,
|
pub binary: String,
|
||||||
|
/// ASan+LSan (or ASan+UBSan for C/C++) build command; `{out}`
|
||||||
|
/// substituted. Run on every would-be accept (SPEC §8). If absent,
|
||||||
|
/// an accept cannot be sanitizer-verified and is capped at
|
||||||
|
/// needs-human-review.
|
||||||
|
#[serde(default)]
|
||||||
|
pub sanitizer: Option<String>,
|
||||||
|
/// Sanitizer binary path with `{out}` substituted.
|
||||||
|
#[serde(default)]
|
||||||
|
pub sanitizer_binary: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Substitute the `{out}` isolation-dir placeholder in a build command or
|
||||||
|
/// binary-path template.
|
||||||
|
pub fn subst_out(template: &str, out_dir: &str) -> String {
|
||||||
|
template.replace("{out}", out_dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
|
|||||||
+11
-10
@@ -339,29 +339,30 @@ fn handle(root: &Path, line: &str) -> Result<Value> {
|
|||||||
)?)?;
|
)?)?;
|
||||||
let t = pending["target"].as_str().unwrap();
|
let t = pending["target"].as_str().unwrap();
|
||||||
let spec = diff_test::target::TargetSpec::load(root, t)?;
|
let spec = diff_test::target::TargetSpec::load(root, t)?;
|
||||||
let bin = Path::new(&spec.build.binary)
|
|
||||||
.file_name()
|
|
||||||
.unwrap()
|
|
||||||
.to_str()
|
|
||||||
.unwrap();
|
|
||||||
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
let diff_path = root.join(format!("results/pending/{patch_id}.json.diff"));
|
||||||
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
|
std::fs::write(&diff_path, pending["diff"].as_str().unwrap())?;
|
||||||
// The build + verdict pipeline runs for minutes — far past the MCP
|
// The build + verdict pipeline runs for minutes — far past the MCP
|
||||||
// transport's per-call cap. Launch it detached, writing progress to
|
// transport's per-call cap. Launch it detached, writing progress to
|
||||||
// a log; the agent observes the result via the read_verdict op,
|
// a log; the agent observes the result via the read_verdict op,
|
||||||
// which reads the append-only ledger row once written.
|
// which reads the append-only ledger row once written.
|
||||||
let cand_dir = format!("targets/{t}/candidate-{patch_id}");
|
// {out} isolates the candidate build (cargo: CARGO_TARGET_DIR;
|
||||||
let cand_bin = format!("{cand_dir}/release/{bin}");
|
// make: build+copy into it) — language-agnostic, mirrors the
|
||||||
|
// baseline rebuild in verdict.
|
||||||
|
let cand_dir = root
|
||||||
|
.join(format!("targets/{t}/candidate-{patch_id}"))
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned();
|
||||||
|
let cand_build = diff_test::target::subst_out(&spec.build.baseline, &cand_dir);
|
||||||
|
let cand_bin = diff_test::target::subst_out(&spec.build.binary, &cand_dir);
|
||||||
let log = format!("results/pending/{run_id}.log");
|
let log = format!("results/pending/{run_id}.log");
|
||||||
// On any failure (candidate build, gates crash) the trap line
|
// On any failure (candidate build, gates crash) the trap line
|
||||||
// marks the log so read_verdict can report "failed" instead of
|
// marks the log so read_verdict can report "failed" instead of
|
||||||
// leaving the agent polling a verdict that will never exist.
|
// leaving the agent polling a verdict that will never exist.
|
||||||
let script = format!(
|
let script = format!(
|
||||||
"{{ set -e\nCARGO_TARGET_DIR={cand} {build}\ncargo run -q -p verdict -- {tgt} \
|
"{{ set -e\n{build}\ncargo run -q -p verdict -- {tgt} \
|
||||||
--rebuild-baseline --candidate-bin {cbin} --run-id {rid} --playbook-class {cls} \
|
--rebuild-baseline --candidate-bin {cbin} --run-id {rid} --playbook-class {cls} \
|
||||||
--hypothesis {hyp} --hotspot {hs} --patch-file {pf}\n}} || echo HOTPATH-PIPELINE-FAILED\n",
|
--hypothesis {hyp} --hotspot {hs} --patch-file {pf}\n}} || echo HOTPATH-PIPELINE-FAILED\n",
|
||||||
cand = shq(&cand_dir),
|
build = cand_build,
|
||||||
build = spec.build.baseline,
|
|
||||||
tgt = shq(t),
|
tgt = shq(t),
|
||||||
cbin = shq(&cand_bin),
|
cbin = shq(&cand_bin),
|
||||||
rid = shq(run_id),
|
rid = shq(run_id),
|
||||||
|
|||||||
+48
-42
@@ -79,14 +79,14 @@ fn rebuild_pristine_baseline(
|
|||||||
"git stash failed"
|
"git stash failed"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
let out_dir = root
|
||||||
|
.join(format!("targets/{target}/baseline"))
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned();
|
||||||
let build = std::process::Command::new("sh")
|
let build = std::process::Command::new("sh")
|
||||||
.arg("-c")
|
.arg("-c")
|
||||||
.arg(&spec.build.baseline)
|
.arg(diff_test::target::subst_out(&spec.build.baseline, &out_dir))
|
||||||
.current_dir(root)
|
.current_dir(root)
|
||||||
.env(
|
|
||||||
"CARGO_TARGET_DIR",
|
|
||||||
root.join(format!("targets/{target}/baseline")),
|
|
||||||
)
|
|
||||||
.status();
|
.status();
|
||||||
if dirty {
|
if dirty {
|
||||||
anyhow::ensure!(
|
anyhow::ensure!(
|
||||||
@@ -98,13 +98,9 @@ fn rebuild_pristine_baseline(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
anyhow::ensure!(build?.success(), "baseline build failed");
|
anyhow::ensure!(build?.success(), "baseline build failed");
|
||||||
let bin_name = std::path::Path::new(&spec.build.binary)
|
let baseline = diff_test::target::subst_out(&spec.build.binary, &out_dir);
|
||||||
.file_name()
|
|
||||||
.and_then(|n| n.to_str())
|
|
||||||
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
|
||||||
let baseline = format!("targets/{target}/baseline/release/{bin_name}");
|
|
||||||
anyhow::ensure!(
|
anyhow::ensure!(
|
||||||
root.join(&baseline).exists(),
|
std::path::Path::new(&baseline).exists(),
|
||||||
"baseline binary missing: {baseline}"
|
"baseline binary missing: {baseline}"
|
||||||
);
|
);
|
||||||
println!("baseline rebuilt from pristine checkout: {baseline}");
|
println!("baseline rebuilt from pristine checkout: {baseline}");
|
||||||
@@ -117,40 +113,43 @@ fn rebuild_pristine_baseline(
|
|||||||
/// were per-attempt manual) and the human audit had to overturn it — the
|
/// were per-attempt manual) and the human audit had to overturn it — the
|
||||||
/// accept path now runs the check itself and caps flagged wins at
|
/// accept path now runs the check itself and caps flagged wins at
|
||||||
/// needs-human-review. Rejections skip it: they ship nothing.
|
/// needs-human-review. Rejections skip it: they ship nothing.
|
||||||
fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> Result<bool> {
|
/// Returns Some(clean) if the sanitizer gate ran, or None if the target
|
||||||
let ws = root.join(format!("targets/{target}/workspace"));
|
/// declares no sanitizer build (an accept then cannot be verified and is
|
||||||
let asan_dir = root.join(format!("targets/{target}/asan"));
|
/// capped at needs-human-review by the caller).
|
||||||
println!("sanitizers: ASan+LSan build (nightly) of the patched tree…");
|
fn sanitizer_check(
|
||||||
let status = std::process::Command::new("cargo")
|
root: &std::path::Path,
|
||||||
.args([
|
target: &str,
|
||||||
"+nightly",
|
spec: &TargetSpec,
|
||||||
"build",
|
) -> Result<Option<bool>> {
|
||||||
"-q",
|
let (Some(build_tpl), Some(bin_tpl)) = (&spec.build.sanitizer, &spec.build.sanitizer_binary)
|
||||||
"--target",
|
else {
|
||||||
"x86_64-unknown-linux-gnu",
|
println!("sanitizers: target declares no sanitizer build — cannot verify an accept");
|
||||||
])
|
return Ok(None);
|
||||||
.env("RUSTFLAGS", "-Zsanitizer=address")
|
};
|
||||||
.env("CARGO_TARGET_DIR", &asan_dir)
|
let out_dir = root
|
||||||
.current_dir(&ws)
|
.join(format!("targets/{target}/asan"))
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned();
|
||||||
|
println!("sanitizers: ASan build of the patched tree…");
|
||||||
|
let status = std::process::Command::new("sh")
|
||||||
|
.arg("-c")
|
||||||
|
.arg(diff_test::target::subst_out(build_tpl, &out_dir))
|
||||||
|
.current_dir(root)
|
||||||
.status()?;
|
.status()?;
|
||||||
anyhow::ensure!(status.success(), "sanitizer build failed");
|
anyhow::ensure!(status.success(), "sanitizer build failed");
|
||||||
let bin_name = std::path::Path::new(&spec.build.binary)
|
let asan_bin = diff_test::target::subst_out(bin_tpl, &out_dir);
|
||||||
.file_name()
|
let cmd = spec.bench.command.replace("{binary}", &asan_bin);
|
||||||
.and_then(|n| n.to_str())
|
|
||||||
.ok_or_else(|| anyhow::anyhow!("bad build.binary path"))?;
|
|
||||||
let asan_bin = asan_dir.join(format!("x86_64-unknown-linux-gnu/debug/{bin_name}"));
|
|
||||||
let cmd = spec
|
|
||||||
.bench
|
|
||||||
.command
|
|
||||||
.replace("{binary}", asan_bin.to_str().unwrap());
|
|
||||||
let out = std::process::Command::new("sh")
|
let out = std::process::Command::new("sh")
|
||||||
.arg("-c")
|
.arg("-c")
|
||||||
.arg(&cmd)
|
.arg(&cmd)
|
||||||
|
// detect_leaks is a no-op for the C/C++ UBSan combo but harmless.
|
||||||
.env("ASAN_OPTIONS", "detect_leaks=1")
|
.env("ASAN_OPTIONS", "detect_leaks=1")
|
||||||
|
.env("UBSAN_OPTIONS", "halt_on_error=1:print_stacktrace=1")
|
||||||
.current_dir(root)
|
.current_dir(root)
|
||||||
.output()?;
|
.output()?;
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||||
let clean = out.status.success() && !stderr.contains("ERROR:");
|
let clean =
|
||||||
|
out.status.success() && !stderr.contains("ERROR:") && !stderr.contains("runtime error:");
|
||||||
if clean {
|
if clean {
|
||||||
println!("sanitizers: clean");
|
println!("sanitizers: clean");
|
||||||
} else {
|
} else {
|
||||||
@@ -161,7 +160,7 @@ fn sanitizer_check(root: &std::path::Path, target: &str, spec: &TargetSpec) -> R
|
|||||||
println!(" {l}");
|
println!(" {l}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(clean)
|
Ok(Some(clean))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn now_utc() -> String {
|
fn now_utc() -> String {
|
||||||
@@ -265,11 +264,18 @@ fn main() -> Result<()> {
|
|||||||
v = Verdict::NeedsHumanReview;
|
v = Verdict::NeedsHumanReview;
|
||||||
}
|
}
|
||||||
if v == Verdict::Accepted {
|
if v == Verdict::Accepted {
|
||||||
let clean = sanitizer_check(&root, &cli.target, &spec)?;
|
match sanitizer_check(&root, &cli.target, &spec)? {
|
||||||
gate_results.sanitizers_clean = clean;
|
Some(true) => gate_results.sanitizers_clean = true,
|
||||||
if !clean {
|
Some(false) => {
|
||||||
println!("verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)");
|
println!(
|
||||||
v = Verdict::NeedsHumanReview;
|
"verdict: sanitizer flag caps the accept at needs-human-review (SPEC §8)"
|
||||||
|
);
|
||||||
|
v = Verdict::NeedsHumanReview;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
println!("verdict: no sanitizer build declared; capping accept at needs-human-review (SPEC §8)");
|
||||||
|
v = Verdict::NeedsHumanReview;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let fp = EnvFingerprint::collect(pin, "system-default");
|
let fp = EnvFingerprint::collect(pin, "system-default");
|
||||||
|
|||||||
@@ -56,3 +56,12 @@ agent-attempt target run_id max_turns="40":
|
|||||||
# Verify the OS boundary from the agent user's side (negative + positive)
|
# Verify the OS boundary from the agent user's side (negative + positive)
|
||||||
isolation-check:
|
isolation-check:
|
||||||
sh scripts/isolation-check.sh
|
sh scripts/isolation-check.sh
|
||||||
|
|
||||||
|
# Install the coz causal profiler (apt, else source build)
|
||||||
|
install-coz:
|
||||||
|
sh scripts/install-coz.sh
|
||||||
|
|
||||||
|
# Causal profile of a C/C++ target: where a speedup would raise throughput
|
||||||
|
coz target iters="200":
|
||||||
|
sh scripts/coz-profile.sh {{target}} {{iters}}
|
||||||
|
python3 scripts/coz-summary.py results/{{target}}/coz/profile.coz
|
||||||
|
|||||||
Binary file not shown.
@@ -0,0 +1,41 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# coz causal profiling for a C/C++ target (SPEC §12). Unlike a flat
|
||||||
|
# profiler (callgrind ranks where time is SPENT), coz measures where a
|
||||||
|
# speedup would actually RAISE throughput — it virtually speeds up each
|
||||||
|
# line and reports the causal effect on the harness's progress points.
|
||||||
|
# Use it to rank playbook targets when the flat profile is ambiguous.
|
||||||
|
#
|
||||||
|
# scripts/coz-profile.sh cjson [iters]
|
||||||
|
#
|
||||||
|
# Requires: coz on PATH (build: git clone github.com/plasma-umass/coz &&
|
||||||
|
# see scripts/install-coz.sh) and the target's harness built with
|
||||||
|
# -DHOTPATH_COZ (this script does that). Output: results/<target>/coz/
|
||||||
|
set -eu
|
||||||
|
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||||
|
TARGET=${1:?usage: coz-profile.sh <target> [iters]}
|
||||||
|
ITERS=${2:-200}
|
||||||
|
OUT="$ROOT/results/$TARGET/coz"
|
||||||
|
mkdir -p "$OUT"
|
||||||
|
|
||||||
|
command -v coz >/dev/null 2>&1 || {
|
||||||
|
echo "coz not on PATH — run scripts/install-coz.sh first" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# coz needs debug line info and the progress point compiled in.
|
||||||
|
BIN="$OUT/cjson-bench-coz"
|
||||||
|
clang -O2 -g -gdwarf-4 -DHOTPATH_COZ \
|
||||||
|
-I "$ROOT/targets/$TARGET/workspace" \
|
||||||
|
-I "$(dirname "$(command -v coz)")/../include" \
|
||||||
|
-o "$BIN" \
|
||||||
|
"$ROOT/targets/$TARGET/harness.c" \
|
||||||
|
"$ROOT/targets/$TARGET/workspace/cJSON.c" \
|
||||||
|
-ldl
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
coz run --output "$OUT/profile.coz" --- \
|
||||||
|
"$BIN" corpora/$TARGET/input/big.json "$ITERS" >/dev/null
|
||||||
|
|
||||||
|
echo "coz profile written: $OUT/profile.coz"
|
||||||
|
echo "view at https://plasma-umass.org/coz/ (load the .coz file) or:"
|
||||||
|
echo " python3 scripts/coz-summary.py $OUT/profile.coz"
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Summarize a coz causal profile: rank source lines by their estimated
|
||||||
|
causal effect on throughput. coz writes newline-delimited JSON records;
|
||||||
|
for each (line, virtual-speedup) it records progress-point throughput.
|
||||||
|
We fit the slope of throughput-gain vs virtual-speedup per line — a
|
||||||
|
steeper positive slope means "speeding up this line most raises overall
|
||||||
|
throughput", which is exactly the class-selection signal a flat profiler
|
||||||
|
can't give.
|
||||||
|
|
||||||
|
python3 scripts/coz-summary.py results/<target>/coz/profile.coz
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
path = Path(sys.argv[1]) if len(sys.argv) > 1 else None
|
||||||
|
if not path or not path.exists():
|
||||||
|
print("usage: coz-summary.py <profile.coz>", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
# experiments: line -> list of (speedup, progress-rate)
|
||||||
|
exps: dict[str, list[tuple[float, float]]] = collections.defaultdict(list)
|
||||||
|
baseline: dict[str, float] = {}
|
||||||
|
for raw in path.read_text().splitlines():
|
||||||
|
raw = raw.strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
rec = json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if rec.get("type") != "experiment":
|
||||||
|
continue
|
||||||
|
line = rec.get("selected", "?")
|
||||||
|
speedup = float(rec.get("speedup", 0.0))
|
||||||
|
duration = float(rec.get("duration", 0.0))
|
||||||
|
delta = float(rec.get("delta", 0.0))
|
||||||
|
if delta <= 0:
|
||||||
|
continue
|
||||||
|
rate = duration / delta # time per progress unit; lower = faster
|
||||||
|
exps[line].append((speedup, rate))
|
||||||
|
if speedup == 0.0:
|
||||||
|
baseline[line] = rate
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
for line, pts in exps.items():
|
||||||
|
base = baseline.get(line)
|
||||||
|
if not base or len(pts) < 2:
|
||||||
|
continue
|
||||||
|
# % throughput improvement at the max virtual speedup tried
|
||||||
|
pts.sort()
|
||||||
|
s_max, r_max = pts[-1]
|
||||||
|
if s_max <= 0 or r_max <= 0:
|
||||||
|
continue
|
||||||
|
gain = (base - r_max) / base # fractional throughput gain
|
||||||
|
rows.append((gain, s_max, line))
|
||||||
|
|
||||||
|
rows.sort(reverse=True)
|
||||||
|
print(f"{'throughput_gain':>16} {'@speedup':>9} line")
|
||||||
|
print("-" * 70)
|
||||||
|
for gain, s_max, line in rows[:25]:
|
||||||
|
print(f"{gain * 100:>14.1f}% {s_max * 100:>7.0f}% {line}")
|
||||||
|
if not rows:
|
||||||
|
print("(no usable experiments — did the run mark progress points?)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Differential fuzz for cjson attempts: pristine vs candidate harness must
|
||||||
|
produce byte-identical stdout (serialization + checksum) AND identical exit
|
||||||
|
code on mutated JSON. cJSON is a deterministic single-threaded parser, so
|
||||||
|
raw byte comparison needs no canonicalization.
|
||||||
|
|
||||||
|
python3 scripts/diff-fuzz-cjson.py <workdir> <iters> <candidate-bin>
|
||||||
|
|
||||||
|
Seeds: slices of the pinned corpus plus hand JSON fragments. Mutations
|
||||||
|
target parser edges: structural tokens, string escapes, number forms
|
||||||
|
(leading zeros, +/- exponents, huge/deep values), and raw byte noise.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import random
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path("/home/user/peltier.io")
|
||||||
|
BASE = ROOT / "targets/cjson/baseline/cjson-bench"
|
||||||
|
CAND = ROOT / sys.argv[3] if len(sys.argv) > 3 else ROOT / "targets/cjson/baseline/cjson-bench"
|
||||||
|
CORPUS = ROOT / "corpora/cjson/input/big.json"
|
||||||
|
WORK = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/tmp/difffuzz-cjson")
|
||||||
|
ITERS = int(sys.argv[2]) if len(sys.argv) > 2 else 10000
|
||||||
|
|
||||||
|
TOKENS = [
|
||||||
|
b"{", b"}", b"[", b"]", b'"', b":", b",", b"\\", b"\\n", b"\\u00e9", b"\\uD83D",
|
||||||
|
b"true", b"false", b"null", b"0", b"-0", b"1e999", b"-1.5E-3", b"00", b"1.",
|
||||||
|
b".5", b"1e", b"NaN", b"Infinity", b'\\"', b'"key"', b"[]", b"{}", b'""',
|
||||||
|
b"1234567890123456789012345", b"\x00", b"\xff", b"\t", b"\n",
|
||||||
|
]
|
||||||
|
|
||||||
|
SEEDS_STATIC = [
|
||||||
|
b'{}', b'[]', b'{"a":1}', b'[1,2,3]', b'"hello"', b'true', b'null',
|
||||||
|
b'{"n":-1.5e10,"s":"a\\tb","u":"\\u00e9","arr":[null,false,{}]}',
|
||||||
|
b'[[[[[[1]]]]]]', b'{"a":{"b":{"c":{"d":1}}}}', b'123.456e-78',
|
||||||
|
b'{"esc":"\\"\\\\\\/\\b\\f\\n\\r\\t"}', b' { "x" : [ 1 , 2 ] } ',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def mutate(data: bytes, rng: random.Random) -> bytes:
|
||||||
|
b = bytearray(data)
|
||||||
|
for _ in range(rng.randrange(1, 8)):
|
||||||
|
if not b:
|
||||||
|
b = bytearray(rng.choice(TOKENS))
|
||||||
|
continue
|
||||||
|
op = rng.randrange(5)
|
||||||
|
pos = rng.randrange(len(b) + 1)
|
||||||
|
if op == 0:
|
||||||
|
b[pos:pos] = rng.choice(TOKENS)
|
||||||
|
elif op == 1 and len(b) > 2:
|
||||||
|
del b[pos:min(len(b), pos + rng.randrange(1, 12))]
|
||||||
|
elif op == 2 and pos < len(b):
|
||||||
|
b[pos] = rng.randrange(256)
|
||||||
|
elif op == 3:
|
||||||
|
end = min(len(b), pos + rng.randrange(1, 48))
|
||||||
|
b[end:end] = b[pos:end]
|
||||||
|
else:
|
||||||
|
b[pos:pos] = rng.choice(TOKENS)
|
||||||
|
return bytes(b)
|
||||||
|
|
||||||
|
|
||||||
|
def run(binary: Path, path: Path) -> tuple[bytes, int]:
|
||||||
|
r = subprocess.run([str(binary), str(path), "1"], capture_output=True, timeout=30)
|
||||||
|
return r.stdout, r.returncode
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
rng = random.Random(0x1CE_B00C)
|
||||||
|
corpus = CORPUS.read_bytes()
|
||||||
|
seeds = list(SEEDS_STATIC)
|
||||||
|
# windows of the real corpus (often invalid mid-slice — exercises the
|
||||||
|
# error path, which must diverge identically or not at all)
|
||||||
|
for o in range(0, len(corpus) - 4096, len(corpus) // 200):
|
||||||
|
for n in (64, 512, 4096):
|
||||||
|
seeds.append(corpus[o : o + n])
|
||||||
|
WORK.mkdir(parents=True, exist_ok=True)
|
||||||
|
case = WORK / "case.json"
|
||||||
|
divergences = 0
|
||||||
|
for i in range(ITERS):
|
||||||
|
payload = mutate(rng.choice(seeds), rng)
|
||||||
|
case.write_bytes(payload)
|
||||||
|
a_out, a_rc = run(BASE, case)
|
||||||
|
b_out, b_rc = run(CAND, case)
|
||||||
|
if a_out != b_out or a_rc != b_rc:
|
||||||
|
divergences += 1
|
||||||
|
keep = WORK / f"divergence-{i}"
|
||||||
|
keep.mkdir(exist_ok=True)
|
||||||
|
(keep / "input.json").write_bytes(payload)
|
||||||
|
(keep / "baseline.out").write_bytes(a_out + f"\nrc={a_rc}".encode())
|
||||||
|
(keep / "candidate.out").write_bytes(b_out + f"\nrc={b_rc}".encode())
|
||||||
|
print(f"DIVERGENCE at iter {i} (rc {a_rc} vs {b_rc}); kept in {keep}")
|
||||||
|
if (i + 1) % 1000 == 0:
|
||||||
|
print(f"{i + 1}/{ITERS}, {divergences} divergences", flush=True)
|
||||||
|
print(f"done: {ITERS} inputs, {divergences} divergences")
|
||||||
|
return 1 if divergences else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Install the coz causal profiler (SPEC §12). Tries the distro package
|
||||||
|
# first, then a source build. Idempotent — skips if `coz` is already on
|
||||||
|
# PATH. Run as a user that can apt-install (root in the dev container).
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if command -v coz >/dev/null 2>&1; then
|
||||||
|
echo "coz already installed: $(command -v coz)"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "installing coz…"
|
||||||
|
if apt-get install -y coz-profiler >/dev/null 2>&1; then
|
||||||
|
echo "installed via apt: $(command -v coz)"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Source build fallback.
|
||||||
|
apt-get install -y python3 libelf-dev libdwarf-dev build-essential git >/dev/null 2>&1 || true
|
||||||
|
SRC=/opt/coz-src
|
||||||
|
rm -rf "$SRC"
|
||||||
|
git clone --depth 1 https://github.com/plasma-umass/coz.git "$SRC"
|
||||||
|
make -C "$SRC" >/dev/null
|
||||||
|
make -C "$SRC" install >/dev/null
|
||||||
|
command -v coz >/dev/null 2>&1 && echo "installed from source: $(command -v coz)" || {
|
||||||
|
echo "coz install failed" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
/* hotpath bench/golden driver for the cjson target.
|
||||||
|
*
|
||||||
|
* TRUST-LAYER CODE — lives outside targets/cjson/workspace, so the agent's
|
||||||
|
* propose_patch allowlist (workspace-relative paths only) cannot touch it.
|
||||||
|
* It is compiled together with the (patched) workspace cJSON.c, so a patch
|
||||||
|
* changes the parser/printer under test but never this measurement rig.
|
||||||
|
*
|
||||||
|
* Workload: parse a JSON document `iters` times, serialize it unformatted
|
||||||
|
* each time, and fold every serialized byte into a rolling checksum so the
|
||||||
|
* optimizer cannot elide the work. Deterministic: the last serialization
|
||||||
|
* plus the checksum are written to stdout, giving golden-replay a
|
||||||
|
* byte-exact signal over both the parse AND the print path.
|
||||||
|
*
|
||||||
|
* cjson-bench <file.json> [iters=40]
|
||||||
|
*/
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include "cJSON.h"
|
||||||
|
|
||||||
|
/* coz causal profiling (SPEC §12): compile with -DHOTPATH_COZ and link
|
||||||
|
* libcoz to mark each parse+print iteration as a throughput progress
|
||||||
|
* point, so `coz run` can measure which source lines, if sped up, would
|
||||||
|
* actually raise end-to-end throughput. A no-op in normal builds. */
|
||||||
|
#ifdef HOTPATH_COZ
|
||||||
|
#include <coz.h>
|
||||||
|
#define HOTPATH_PROGRESS() COZ_PROGRESS
|
||||||
|
#else
|
||||||
|
#define HOTPATH_PROGRESS() ((void)0)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
int main(int argc, char **argv) {
|
||||||
|
if (argc < 2) {
|
||||||
|
fprintf(stderr, "usage: %s <file.json> [iters]\n", argv[0]);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
long iters = (argc >= 3) ? atol(argv[2]) : 40;
|
||||||
|
if (iters < 1) iters = 1;
|
||||||
|
|
||||||
|
FILE *f = fopen(argv[1], "rb");
|
||||||
|
if (!f) { perror("fopen"); return 2; }
|
||||||
|
fseek(f, 0, SEEK_END);
|
||||||
|
long n = ftell(f);
|
||||||
|
fseek(f, 0, SEEK_SET);
|
||||||
|
if (n < 0) { fclose(f); return 2; }
|
||||||
|
char *buf = (char *)malloc((size_t)n + 1);
|
||||||
|
if (!buf) { fclose(f); return 2; }
|
||||||
|
if (fread(buf, 1, (size_t)n, f) != (size_t)n) { free(buf); fclose(f); return 2; }
|
||||||
|
buf[n] = '\0';
|
||||||
|
fclose(f);
|
||||||
|
|
||||||
|
unsigned long acc = 1469598103934665603UL; /* FNV-ish rolling checksum */
|
||||||
|
char *last = NULL;
|
||||||
|
for (long i = 0; i < iters; i++) {
|
||||||
|
cJSON *root = cJSON_Parse(buf);
|
||||||
|
if (!root) { fprintf(stderr, "parse failed\n"); free(buf); return 3; }
|
||||||
|
char *out = cJSON_PrintUnformatted(root);
|
||||||
|
if (!out) { cJSON_Delete(root); free(buf); return 3; }
|
||||||
|
for (char *p = out; *p; p++) acc = (acc ^ (unsigned char)*p) * 1099511628211UL;
|
||||||
|
if (i == iters - 1) last = out;
|
||||||
|
else free(out);
|
||||||
|
cJSON_Delete(root);
|
||||||
|
HOTPATH_PROGRESS(); /* one parse+print = one throughput unit (coz) */
|
||||||
|
}
|
||||||
|
|
||||||
|
if (last) {
|
||||||
|
fwrite(last, 1, strlen(last), stdout);
|
||||||
|
free(last);
|
||||||
|
}
|
||||||
|
printf("\nchecksum=%lu\n", acc);
|
||||||
|
free(buf);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# cJSON — ultralightweight JSON parser/printer (C)
|
||||||
|
# Selection vs SPEC §6: CPU-bound parse+serialize; hermetic Unity/CTest
|
||||||
|
# suite vendored in-tree; deterministic scripted workload; MIT license;
|
||||||
|
# active but deliberately "simple", no dedicated perf team — the richer
|
||||||
|
# legacy-C hunting ground Phase 3 calls for.
|
||||||
|
|
||||||
|
[source]
|
||||||
|
repo = "https://github.com/DaveGamble/cJSON.git"
|
||||||
|
commit = "c859b25da02955fef659d658b8f324b5cde87be3" # v1.7.19
|
||||||
|
license = "MIT"
|
||||||
|
|
||||||
|
[fetch]
|
||||||
|
# targets/*/workspace is gitignored; reproduce it with:
|
||||||
|
# git clone https://github.com/DaveGamble/cJSON.git workspace
|
||||||
|
# git -C workspace checkout v1.7.19
|
||||||
|
# The bench/golden driver targets/cjson/harness.c and the Unity suite in
|
||||||
|
# workspace/tests/ (unity vendored in-tree) are hermetic — no submodules.
|
||||||
|
|
||||||
|
[build]
|
||||||
|
# Compile the trust-layer harness against the (patched) workspace cJSON.c.
|
||||||
|
# {out} is the isolation dir; the agent may patch only workspace/*, never
|
||||||
|
# harness.c, so the measurement rig is fixed across baseline/candidate.
|
||||||
|
baseline = "mkdir -p {out} && clang -O2 -DNDEBUG -I targets/cjson/workspace -o {out}/cjson-bench targets/cjson/harness.c targets/cjson/workspace/cJSON.c"
|
||||||
|
binary = "{out}/cjson-bench"
|
||||||
|
# ASan + UBSan (C/C++ analogue of the Rust nightly ASan path); run on
|
||||||
|
# every would-be accept (SPEC §8), halting on any leak/UB.
|
||||||
|
sanitizer = "mkdir -p {out} && clang -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -I targets/cjson/workspace -o {out}/cjson-asan targets/cjson/harness.c targets/cjson/workspace/cJSON.c"
|
||||||
|
sanitizer_binary = "{out}/cjson-asan"
|
||||||
|
|
||||||
|
[bench]
|
||||||
|
command = "{binary} corpora/cjson/input/big.json 2 > /dev/null"
|
||||||
|
workload = "5.6 MB synthetic JSON (20k records: nested objects/arrays, escaped/unicode strings, ints, dense floats), parsed+serialized 2x, single thread"
|
||||||
|
|
||||||
|
[gates]
|
||||||
|
# Upstream Unity suite via CTest — parse_number/string/array/object,
|
||||||
|
# print_*, misc_tests, compare/minify. Tests the PATCHED cJSON.c.
|
||||||
|
tests = "cmake -S targets/cjson/workspace -B targets/cjson/testbuild -DENABLE_CJSON_TEST=On -DCMAKE_BUILD_TYPE=Debug >/dev/null 2>&1 && cmake --build targets/cjson/testbuild -j >/dev/null 2>&1 && ctest --test-dir targets/cjson/testbuild --output-on-failure"
|
||||||
|
# Golden: full re-serialization + checksum of the corpus, byte-hashed.
|
||||||
|
golden = "{binary} corpora/cjson/input/big.json 1"
|
||||||
|
|
||||||
|
[corpus]
|
||||||
|
manifest = "corpora/cjson/MANIFEST.sha256"
|
||||||
|
root = "corpora/cjson/input"
|
||||||
|
golden_sha256 = "corpora/cjson/GOLDEN.sha256"
|
||||||
@@ -18,8 +18,10 @@ license = "BSD-2-Clause"
|
|||||||
# git clone <submodule repo> workspace/vendor/progit && git -C ... checkout <commit>
|
# git clone <submodule repo> workspace/vendor/progit && git -C ... checkout <commit>
|
||||||
|
|
||||||
[build]
|
[build]
|
||||||
baseline = "cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml"
|
baseline = "CARGO_TARGET_DIR={out} cargo build --release --manifest-path targets/comrak/workspace/Cargo.toml"
|
||||||
binary = "targets/comrak/workspace/target/release/comrak"
|
binary = "{out}/release/comrak"
|
||||||
|
sanitizer = "CARGO_TARGET_DIR={out} RUSTFLAGS='-Zsanitizer=address' cargo +nightly build --target x86_64-unknown-linux-gnu --manifest-path targets/comrak/workspace/Cargo.toml"
|
||||||
|
sanitizer_binary = "{out}/x86_64-unknown-linux-gnu/debug/comrak"
|
||||||
|
|
||||||
[bench]
|
[bench]
|
||||||
# Upstream workload (benches/bench.sh): full Pro Git book through the
|
# Upstream workload (benches/bench.sh): full Pro Git book through the
|
||||||
|
|||||||
@@ -9,8 +9,10 @@ commit = "fa44e5194060305576514d59b850353643afbfc8" # v14.0.0
|
|||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|
||||||
[build]
|
[build]
|
||||||
baseline = "cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml"
|
baseline = "CARGO_TARGET_DIR={out} cargo build --release --manifest-path targets/tokei/workspace/Cargo.toml"
|
||||||
binary = "targets/tokei/workspace/target/release/tokei"
|
binary = "{out}/release/tokei"
|
||||||
|
sanitizer = "CARGO_TARGET_DIR={out} RUSTFLAGS='-Zsanitizer=address' cargo +nightly build --target x86_64-unknown-linux-gnu --manifest-path targets/tokei/workspace/Cargo.toml"
|
||||||
|
sanitizer_binary = "{out}/x86_64-unknown-linux-gnu/debug/tokei"
|
||||||
|
|
||||||
[bench]
|
[bench]
|
||||||
# Single rayon thread: measurement first (SPEC conventions); the
|
# Single rayon thread: measurement first (SPEC conventions); the
|
||||||
|
|||||||
Reference in New Issue
Block a user