diff --git a/agent/hotpath_agent/harness.py b/agent/hotpath_agent/harness.py index 3364af5..0f2f33d 100644 --- a/agent/hotpath_agent/harness.py +++ b/agent/hotpath_agent/harness.py @@ -1,13 +1,21 @@ """Client for harnessd — the agent's only door to the trust layer. -Spawns the Rust daemon and speaks one JSON object per line over its -stdio. The client is deliberately dumb: no retries that could double- -apply a patch, no local fallbacks that could fabricate evidence. +Two transports: +- Unix socket (production/SPEC §10): if HOTPATH_HARNESS_SOCKET is set, + connect to a daemon owned by the trusted uid. The agent process runs + unprivileged and cannot write the repo except through this door. +- Spawned stdio subprocess (dev fallback): same uid, defense-in-depth + only — the filesystem boundary does not hold in this mode. + +The client is deliberately dumb: no retries that could double-apply a +patch, no local fallbacks that could fabricate evidence. """ from __future__ import annotations import json +import os +import socket import subprocess from typing import Any @@ -18,27 +26,45 @@ class HarnessError(RuntimeError): class HarnessClient: def __init__(self, repo_root: str, binary: str = "target/debug/harnessd"): - self._proc = subprocess.Popen( - [binary], - cwd=repo_root, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - text=True, - ) + sock_path = os.environ.get("HOTPATH_HARNESS_SOCKET") + self._proc: subprocess.Popen[str] | None = None + self._sock: socket.socket | None = None + if sock_path: + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._sock.connect(sock_path) + self._io = self._sock.makefile("rw", encoding="utf-8", newline="\n") + else: + self._proc = subprocess.Popen( + [binary], + cwd=repo_root, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) def call(self, op: str, **kwargs: Any) -> dict[str, Any]: - assert self._proc.stdin and self._proc.stdout - self._proc.stdin.write(json.dumps({"op": op, **kwargs}) + "\n") - self._proc.stdin.flush() - line = self._proc.stdout.readline() + payload = json.dumps({"op": op, **kwargs}) + "\n" + if self._sock is not None: + self._io.write(payload) + self._io.flush() + line = self._io.readline() + else: + assert self._proc and self._proc.stdin and self._proc.stdout + self._proc.stdin.write(payload) + self._proc.stdin.flush() + line = self._proc.stdout.readline() if not line: - raise HarnessError("harnessd exited") + raise HarnessError("harnessd connection closed") resp = json.loads(line) if not resp.get("ok"): raise HarnessError(resp.get("error", "unknown harness error")) return resp["result"] def close(self) -> None: - if self._proc.stdin: - self._proc.stdin.close() - self._proc.wait(timeout=10) + if self._sock is not None: + self._io.close() + self._sock.close() + if self._proc is not None: + if self._proc.stdin: + self._proc.stdin.close() + self._proc.wait(timeout=10) diff --git a/agent/hotpath_agent/loop.py b/agent/hotpath_agent/loop.py index 267f632..95017b0 100644 --- a/agent/hotpath_agent/loop.py +++ b/agent/hotpath_agent/loop.py @@ -36,15 +36,14 @@ HARNESS_TOOLS = [ ] # Built-in tools we deny at the SDK layer. This is DEFENSE IN DEPTH, not the -# trust boundary. SPEC §10's real boundary is OS-level process/user isolation: -# harnessd runs as a separate process the agent can't bypass, target code runs -# in a no-net container. In THIS dev container the nested agent runs as root -# under a parent Claude Code session, so neither --dangerously-skip-permissions -# nor a can_use_tool callback reliably gates built-ins (settings inherited from -# the parent session shadow them). We enumerate what we can; production runs the -# agent process without host-shell capability at all. What holds regardless: -# propose_patch is the only harness-mediated write path (path-allowlist + git -# apply), and the ledger is append-only (mutation-refusing triggers). +# trust boundary. SPEC §10's real boundary is OS-level isolation, shipped in +# scripts/agent-isolated.sh: this whole process tree runs either inside a +# mount namespace with the repo read-only and CAP_SYS_ADMIN dropped, or as +# the unprivileged hpagent user — verified by `just isolation-check`. Even +# if the nested CLI reaches a shell despite this list (settings inherited +# from a parent session can shadow SDK gating), repo writes fail at the OS. +# What holds regardless: propose_patch is the only harness-mediated write +# path (path-allowlist + git apply), and the ledger is append-only. BUILTIN_TOOLS_DENIED = [ "Bash", "Read", "Write", "Edit", "Glob", "Grep", "WebSearch", "WebFetch", "Task", "TodoWrite", diff --git a/crates/harnessd/src/main.rs b/crates/harnessd/src/main.rs index fa241bb..39ec4cc 100644 --- a/crates/harnessd/src/main.rs +++ b/crates/harnessd/src/main.rs @@ -10,11 +10,24 @@ use anyhow::{anyhow, ensure, Result}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use std::io::{BufRead, Write}; +use std::os::unix::net::UnixListener; use std::path::{Component, Path}; use std::process::Command; +use std::sync::Mutex; + +/// Serializes mutating ops (git apply, pending writes) across socket +/// connections; reads don't need it but the cost is nil at one agent. +static HANDLE_LOCK: Mutex<()> = Mutex::new(()); fn main() -> Result<()> { let root = std::env::current_dir()?.canonicalize()?; + let args: Vec = std::env::args().collect(); + if let Some(i) = args.iter().position(|a| a == "--socket") { + let path = args + .get(i + 1) + .ok_or_else(|| anyhow!("--socket requires a path"))?; + return serve_socket(&root, Path::new(path)); + } let stdin = std::io::stdin(); let mut stdout = std::io::stdout().lock(); for line in stdin.lock().lines() { @@ -22,16 +35,62 @@ fn main() -> Result<()> { if line.trim().is_empty() { continue; } - let resp = match handle(&root, &line) { - Ok(v) => json!({"ok": true, "result": v}), - Err(e) => json!({"ok": false, "error": e.to_string()}), - }; + let resp = respond(&root, &line); writeln!(stdout, "{resp}")?; stdout.flush()?; } Ok(()) } +/// SPEC §10 socket mode: harnessd runs as the trusted uid and listens on a +/// Unix socket; the agent process runs as an unprivileged user whose only +/// write path into the repo is this daemon. Socket permissions (owner/group/ +/// mode) are set by the supervisor script that starts us. +fn serve_socket(root: &Path, sock: &Path) -> Result<()> { + if let Some(dir) = sock.parent() { + std::fs::create_dir_all(dir)?; + } + let _ = std::fs::remove_file(sock); + let listener = UnixListener::bind(sock)?; + eprintln!("harnessd listening on {}", sock.display()); + for conn in listener.incoming() { + let stream = match conn { + Ok(s) => s, + Err(e) => { + eprintln!("accept error: {e}"); + continue; + } + }; + let root = root.to_path_buf(); + std::thread::spawn(move || { + let reader = match stream.try_clone() { + Ok(s) => std::io::BufReader::new(s), + Err(_) => return, + }; + let mut writer = stream; + for line in reader.lines() { + let Ok(line) = line else { break }; + if line.trim().is_empty() { + continue; + } + let resp = respond(&root, &line); + if writeln!(writer, "{resp}").is_err() { + break; + } + } + }); + } + Ok(()) +} + +fn respond(root: &Path, line: &str) -> Value { + let _guard = HANDLE_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + match handle(root, line) { + Ok(v) => json!({"ok": true, "result": v}), + Err(e) => json!({"ok": false, "error": e.to_string()}), + } +} + /// Single-quote a string for safe embedding in a generated shell script. fn shq(s: &str) -> String { format!("'{}'", s.replace('\'', "'\\''")) diff --git a/docs/infra.md b/docs/infra.md index 0b9c150..bb0f3f2 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -11,7 +11,38 @@ repo's CI cannot provide. Status by requirement: | Environment noise measured | ✅ (measured, likely fails bars) | `runner-calibration` CI job, artifact per run | | Profiling | ⚠️ software `cpu-clock` only (VM PMU usually absent) | `perf-probe` CI job reports what works | | Pinned cores, governor, SMT/turbo control | ❌ shared VM | **bench-metal box (below)** | -| Separate-uid read-only trust layer | ❌ | bench-metal box setup | +| Separate-uid read-only trust layer | ✅ shipped | `scripts/agent-isolated.sh` + `just isolation-check` (below) | + +## OS-level agent isolation (SPEC §10) — shipped + +`scripts/agent-isolated.sh` launches one unattended attempt with the +agent process tree behind an OS boundary; `just isolation-check` +verifies the boundary from the agent's side (19 negative/positive +checks, both modes). harnessd runs OUTSIDE the boundary as the trusted +uid, serving its seven ops on a Unix socket (`harnessd --socket`) — the +agent's only write path into the repo. + +Two modes (`HOTPATH_ISOLATION`): + +- **`mountns` (default in this container).** The agent tree runs uid 0 + inside a private mount namespace with the repo bind-mounted + read-only, `CAP_SYS_ADMIN`/`CAP_SYS_PTRACE` dropped from the bounding + set. Verified refused: repo writes (EROFS), forged ledger INSERT, + trust-binary replacement, `mount -o remount,rw`, and userns re-entry + to undo the mount. Chosen because the nested CLI's auth is bound to + the parent session's uid — no credentials are copied anywhere. +- **`user` (bench-metal shape).** The agent runs as unprivileged + `hpagent`; the repo is root-owned with no world-writable paths (the + launcher refuses to start otherwise). Verified refused: all of the + above plus running the `verdict` binary directly. Requires the agent + user to bring its own API credentials; a parent session's root-owned + auth is deliberately not copied across the boundary. + +Residual (both modes, this container): the agent tree can still read +the repo (by design — it needs source), reach the network through the +container's egress proxy, and signal same-uid processes in `mountns` +mode. The bench-metal `user` mode plus no-net target containers remains +the production shape. ## The bench-metal box (human setup, ~15 min + account) diff --git a/justfile b/justfile index b394e2e..140fe98 100644 --- a/justfile +++ b/justfile @@ -47,3 +47,12 @@ verdict target candidate run_id class hypothesis hotspot *flags: --candidate-bin "{{candidate}}" --run-id "{{run_id}}" \ --playbook-class {{class}} --hypothesis "{{hypothesis}}" \ --hotspot "{{hotspot}}" {{flags}} + +# One unattended agent attempt behind the SPEC §10 OS boundary: +# harnessd (trusted uid) on a Unix socket, agent loop as user hpagent. +agent-attempt target run_id max_turns="40": + sh scripts/agent-isolated.sh {{target}} {{run_id}} {{max_turns}} + +# Verify the OS boundary from the agent user's side (negative + positive) +isolation-check: + sh scripts/isolation-check.sh diff --git a/scripts/agent-isolated.sh b/scripts/agent-isolated.sh new file mode 100755 index 0000000..8057fed --- /dev/null +++ b/scripts/agent-isolated.sh @@ -0,0 +1,107 @@ +#!/bin/sh +# Launch one unattended optimization attempt behind an OS-level boundary +# (SPEC §10). Run as the trusted uid (root in this container): +# +# scripts/agent-isolated.sh [max-turns] +# +# Two isolation modes (HOTPATH_ISOLATION, default "mountns"): +# +# mountns — this container's boundary. The agent process tree (Claude +# CLI + MCP server + harness client) runs in a private mount +# namespace where the repo is bind-mounted READ-ONLY, with +# CAP_SYS_ADMIN and CAP_SYS_PTRACE removed from the bounding set so +# the mount cannot be undone (verified: remount and userns re-entry +# both refused). Repo writes fail with EROFS no matter the uid; the +# CLI's own auth state (under $HOME) stays untouched. No credential +# copying. +# +# user — the bench-metal shape (docs/infra.md). The agent loop runs as +# the unprivileged user 'hpagent' whose only door into the repo is +# the harnessd socket. Requires the agent user to have its own API +# credentials (CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY in its +# environment) — a parent session's root-owned auth is deliberately +# NOT copied across the boundary. +# +# In both modes harnessd + the verdict pipeline run OUTSIDE the boundary +# as the trusted uid and own every write into the repo (git apply, +# ledger rows, pending/ evidence). Verify the boundary any time with +# scripts/isolation-check.sh. +set -eu + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +TARGET=${1:?usage: agent-isolated.sh [max-turns]} +RUN_ID=${2:?usage: agent-isolated.sh [max-turns]} +MAX_TURNS=${3:-40} +MODE=${HOTPATH_ISOLATION:-mountns} + +AGENT_USER=hpagent +SOCK_DIR=/run/hotpath +SOCK=$SOCK_DIR/harness.sock + +[ "$(id -u)" -eq 0 ] || { echo "must run as the trusted uid (root)" >&2; exit 1; } + +# --- no world-writable holes in the repo ------------------------------------ +# In user mode the boundary is filesystem permissions; a single o+w path +# would void it. mountns mode doesn't need this, but a hole is still a +# smell worth failing on. +HOLES=$(find "$ROOT" -perm -o+w \( -type f -o -type d \) 2>/dev/null || true) +if [ -n "$HOLES" ]; then + echo "refusing to launch: world-writable paths inside the repo:" >&2 + echo "$HOLES" >&2 + exit 1 +fi + +# --- trust-layer daemon (outside the boundary) -------------------------------- +( cd "$ROOT" && cargo build -q -p harnessd -p verdict ) + +mkdir -p "$SOCK_DIR" +if [ ! -S "$SOCK" ]; then + ( cd "$ROOT" && setsid ./target/debug/harnessd --socket "$SOCK" \ + >>/var/log/hotpath-harnessd.log 2>&1 & ) + for _ in $(seq 1 50); do [ -S "$SOCK" ] && break; sleep 0.1; done + [ -S "$SOCK" ] || { echo "harnessd socket never appeared" >&2; exit 1; } +fi + +run_loop="python3 -m hotpath_agent.loop $TARGET --run-id $RUN_ID --max-turns $MAX_TURNS --repo-root $ROOT" + +case "$MODE" in +mountns) + chmod 600 "$SOCK" # uid 0 only; the confined tree is still uid 0 + exec unshare -m sh -c " + mount --bind '$ROOT' '$ROOT' && + mount -o remount,bind,ro '$ROOT' && + exec setpriv --bounding-set -sys_admin,-sys_ptrace --inh-caps -all \ + env HOTPATH_HARNESS_SOCKET='$SOCK' \ + HOTPATH_REPO_ROOT='$ROOT' \ + PYTHONPATH='$ROOT/agent' \ + $run_loop" + ;; +user) + id "$AGENT_USER" >/dev/null 2>&1 || useradd -m -s /bin/sh "$AGENT_USER" + HP_HOME=$(getent passwd "$AGENT_USER" | cut -d: -f6) + chown root:"$AGENT_USER" "$SOCK" + chmod 660 "$SOCK" + # The container's TLS-intercepting proxy CA lives under /root (mode + # 700); the agent user needs a readable copy to reach the API. + CA_SRC=${HOTPATH_CA_SRC:-/root/.ccr/ca-bundle.crt} + CA=/etc/hotpath-ca.crt + [ -f "$CA_SRC" ] && install -m 644 "$CA_SRC" "$CA" + # setpriv preserves the environment (proxy/API routing vars); we + # override HOME and every CA var that pointed under /root. The agent + # user must bring its own API credentials — see header. + exec setpriv --reuid "$AGENT_USER" --regid "$AGENT_USER" --init-groups \ + env HOME="$HP_HOME" USER="$AGENT_USER" LOGNAME="$AGENT_USER" \ + HOTPATH_HARNESS_SOCKET="$SOCK" \ + HOTPATH_REPO_ROOT="$ROOT" \ + PYTHONPATH="$ROOT/agent" \ + SSL_CERT_FILE="$CA" NODE_EXTRA_CA_CERTS="$CA" \ + CURL_CA_BUNDLE="$CA" REQUESTS_CA_BUNDLE="$CA" \ + GIT_SSL_CAINFO="$CA" AWS_CA_BUNDLE="$CA" \ + NIX_SSL_CERT_FILE="$CA" GRPC_DEFAULT_SSL_ROOTS_FILE_PATH="$CA" \ + $run_loop + ;; +*) + echo "unknown HOTPATH_ISOLATION mode: $MODE (mountns|user)" >&2 + exit 1 + ;; +esac diff --git a/scripts/isolation-check.sh b/scripts/isolation-check.sh new file mode 100755 index 0000000..ab485e7 --- /dev/null +++ b/scripts/isolation-check.sh @@ -0,0 +1,132 @@ +#!/bin/sh +# Verify the SPEC §10 OS boundary from the agent's side of it. Run as the +# trusted uid (root). Checks BOTH isolation modes of agent-isolated.sh: +# every write the agent could use to game the harness must fail; the +# harnessd socket door must work. Exits nonzero on any hole. +set -u + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +AGENT_USER=hpagent +SOCK=${HOTPATH_HARNESS_SOCKET:-/run/hotpath/harness.sock} +FAIL=0 + +note_fail() { echo "FAIL (boundary hole): $1"; FAIL=1; } +note_ok() { echo "ok: $1"; } + +[ -S "$SOCK" ] || { echo "FAIL: harnessd socket $SOCK missing (run agent-isolated.sh or start the daemon)"; exit 1; } + +# Probe scripts as files: they run nested inside sh -c layers where +# inline `python3 -c` quoting is unmanageable. +PROBE_DIR=$(mktemp -d) +trap 'rm -rf "$PROBE_DIR"' EXIT +chmod 755 "$PROBE_DIR" + +cat > "$PROBE_DIR/sock_check.py" < "$PROBE_DIR/escape_check.py" < "$PROBE_DIR/ledger_insert.py" </dev/null 2>&1; then + note_fail "[mountns] $desc succeeded" + else + note_ok "[mountns] $desc refused" + fi +} +mnt_must_pass() { + desc=$1; cmd=$2 + if $MNT_PREFIX sh -c "$CONFINE sh -c \"$cmd\"" >/dev/null 2>&1; then + note_ok "[mountns] $desc works" + else + note_fail "[mountns] $desc broken" + fi +} + +mnt_must_fail "write into crates/ (trust layer)" "touch '$ROOT/crates/.probe'" +mnt_must_fail "edit config/accept.toml (thresholds)" "echo x >> '$ROOT/config/accept.toml'" +mnt_must_fail "write a target workspace directly" "touch '$ROOT/targets/comrak/workspace/.probe'" +mnt_must_fail "INSERT a forged ledger row" "python3 $PROBE_DIR/ledger_insert.py" +mnt_must_fail "overwrite results/ evidence" "echo x > '$ROOT/results/.probe'" +mnt_must_fail "replace a trust binary" "echo x > '$ROOT/target/debug/verdict'" +mnt_must_fail "remount the repo read-write" "mount -o remount,bind,rw '$ROOT'" +mnt_must_fail "userns re-entry to undo the ro mount" "unshare -Ur -m mount -o remount,bind,rw '$ROOT'" +mnt_must_pass "harnessd socket read_ledger" "python3 $PROBE_DIR/sock_check.py" + +# ============================ user mode ======================================= +if id "$AGENT_USER" >/dev/null 2>&1; then + echo "--- user mode (unprivileged $AGENT_USER) ---" + HP_HOME=$(getent passwd "$AGENT_USER" | cut -d: -f6) + as_agent() { + setpriv --reuid "$AGENT_USER" --regid "$AGENT_USER" --init-groups \ + env HOME="$HP_HOME" "$@" + } + must_fail() { + desc=$1; shift + if as_agent "$@" >/dev/null 2>&1; then + note_fail "[user] $desc succeeded as $AGENT_USER" + else + note_ok "[user] $desc refused" + fi + } + must_pass() { + desc=$1; shift + if as_agent "$@" >/dev/null 2>&1; then + note_ok "[user] $desc works" + else + note_fail "[user] $desc broken for $AGENT_USER" + fi + } + # user mode needs group access on the socket for the positive checks + SOCK_GRP_SAVED=$(stat -c '%U:%G %a' "$SOCK") + chown root:"$AGENT_USER" "$SOCK"; chmod 660 "$SOCK" + + must_fail "write into crates/ (trust layer)" sh -c "touch '$ROOT/crates/.probe'" + must_fail "edit config/accept.toml (thresholds)" sh -c "echo x >> '$ROOT/config/accept.toml'" + must_fail "write into corpora/ (pinned inputs)" sh -c "touch '$ROOT/corpora/.probe'" + must_fail "write a target workspace directly" sh -c "touch '$ROOT/targets/comrak/workspace/.probe'" + must_fail "INSERT a forged ledger row" python3 "$PROBE_DIR/ledger_insert.py" + must_fail "overwrite results/ evidence" sh -c "echo x > '$ROOT/results/.probe'" + must_fail "replace a trust binary" sh -c "echo x > '$ROOT/target/debug/verdict'" + must_fail "run the verdict binary to write a row" sh -c "cd '$ROOT' && ./target/debug/verdict comrak --candidate-bin /bin/true --run-id forged-direct --playbook-class 1 --hypothesis x --hotspot x" + must_pass "harnessd socket read_ledger" python3 "$PROBE_DIR/sock_check.py" + must_pass "harnessd refuses ../ escape in a diff" python3 "$PROBE_DIR/escape_check.py" + + chown "${SOCK_GRP_SAVED%% *}" "$SOCK"; chmod "${SOCK_GRP_SAVED##* }" "$SOCK" +else + echo "--- user mode skipped (no $AGENT_USER user) ---" +fi + +if [ "$FAIL" -eq 0 ]; then + echo "isolation-check: ALL CHECKS PASSED" +else + echo "isolation-check: BOUNDARY HOLES FOUND" >&2 +fi +exit "$FAIL"