diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f8412d..4211180 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,9 @@ jobs: with: node-version: "20" + - name: Conformance vectors match the generator (no drift, frozen v1.0 tips) + run: python3 spec/korg-ledger-v1/tools/generate_vectors.py + - name: Python spec oracle reproduces the frozen vectors run: python3 spec/korg-ledger-v1/conformance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb0fb5..d059df8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,42 @@ versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -_Nothing yet._ +### korg-ledger@v1 revision 1.1 — the canonical flight-recorder contract + +`spec/korg-ledger-v1/` is now **authoritative** for every korg document and +implementation. No hash rule changed — every v1.0 vector reproduces its frozen +tip. Upgrade notes: [`spec/korg-ledger-v1/MIGRATION.md`](spec/korg-ledger-v1/MIGRATION.md). + +**Fixed — causal links in real journals were never checked.** All three +verifiers read `triggered_by` only at the top level, but every production writer +stores it under `metadata.triggered_by`, so forward or dangling links in nested +journals verified VALID. Rust, Python, and JS now resolve the parent from either +shape (SPEC §9), reject non-integer and conflicting links, and emit +byte-identical DAG errors (`seq Some(3)` in Rust is gone). Verdict splits on +`triggered_by: true` (Python-only) and `1.5` (JS-only) are closed. + +**Added — redaction at persistence (`korg-redact@v1`, REDACTION.md).** Secrets +(sensitive keys, bearer tokens, provider/GitHub/Slack/AWS/Google keys, JWTs, PEM +private keys, URL credentials, `NAME_TOKEN=…` assignments) are replaced by +`[REDACTED:]` markers *before* hashing in `LedgerWriter` (default on), +`POST /api/agent/tool-call`, and korg-bridge. Python and Rust implementations are +pinned by shared hand-authored cases and differentially fuzzed. +`korg-verify --require-redacted` and `python -m korg_ledger lint` flag any +persisted secret; `python -m korg_ledger rechain-redacted` migrates a leaked +journal to a new, clean chain. + +**Added — import envelopes (`korg-import@v1`, IMPORT.md).** Allowlisted, +atomic imports for sanitized Okto diagnostics (`okto.diagnostics@v1`) and +ningen-shikkaku lifecycle metadata (`ningen-shikkaku.lifecycle@v1`), with +projection-only exporters from Okto session/genome files and `[dazai]` daemon +logs. Imported events use the new `import:@` actor prefix. + +**Changed.** `POST /api/agent/tool-call` answers **422** for a `triggered_by` +that is not an existing, strictly-earlier `seq_id` (it used to persist it). +`LedgerWriter` rejects non-`int` `triggered_by`. The duplicate vector set in +`crates/korg-registry/tests/conformance/` is removed; all harnesses read +`spec/korg-ledger-v1/`, now with nested, mixed-shape, malformed-link, and +redaction vectors, generated and drift-checked by `tools/generate_vectors.py`. --- diff --git a/Cargo.lock b/Cargo.lock index f44ba4f..e376db5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2055,6 +2055,7 @@ version = "0.3.3" dependencies = [ "chrono", "korg-core", + "korg-ledger", "korg-registry", "pyo3", "pythonize", @@ -2098,6 +2099,7 @@ dependencies = [ "hex", "hmac", "proptest", + "regex", "serde_json", "sha2", ] @@ -2192,6 +2194,7 @@ dependencies = [ "korg-auth", "korg-core", "korg-embeddings", + "korg-ledger", "korg-llm", "korg-registry", "korg-runtime", diff --git a/adapters/korg-ledger-py/README.md b/adapters/korg-ledger-py/README.md index 65ca904..dc9609b 100644 --- a/adapters/korg-ledger-py/README.md +++ b/adapters/korg-ledger-py/README.md @@ -6,7 +6,18 @@ frozen spec at `spec/korg-ledger-v1/`, pinned to the same conformance vectors as the Rust and JavaScript references. `LedgerWriter` produces hash-chained `JournalEvent` JSONL that the Rust -`korg-verify` binary validates byte-for-byte. +`korg-verify` binary validates byte-for-byte. Every record is redacted under +[`korg-redact@v1`](../../spec/korg-ledger-v1/REDACTION.md) **before** it is hashed +or written (opt out only with `redaction=None`). + + python -m korg_ledger lint ledger.jsonl # chain + causal DAG (flat & nested) + redaction + python -m korg_ledger envelope okto-session S.json --source-version 0.1.0 > env.json + python -m korg_ledger envelope dazai-log daemon.log --source-version 0.1.0 > env.json + python -m korg_ledger import env.json --ledger ledger.jsonl # korg-import@v1, atomic on rejection + python -m korg_ledger rechain-redacted old.jsonl new.jsonl # migrate a journal that leaked a secret + +See [`IMPORT.md`](../../spec/korg-ledger-v1/IMPORT.md) for the Okto and +ningen-shikkaku envelope profiles. pip install -e adapters/korg-ledger-py pytest adapters/korg-ledger-py/tests -v diff --git a/adapters/korg-ledger-py/src/korg_ledger/__init__.py b/adapters/korg-ledger-py/src/korg_ledger/__init__.py index 226a003..bd05990 100644 --- a/adapters/korg-ledger-py/src/korg_ledger/__init__.py +++ b/adapters/korg-ledger-py/src/korg_ledger/__init__.py @@ -11,13 +11,17 @@ GENESIS, HASH_FIELDS, canonicalize, + causal_parent, chain_hash, + json_int, verify_anchors, verify_chain, verify_dag, ) from ._hlc import Hlc -from .writer import CausalityError, LedgerWriter +from .redaction import POLICY as REDACTION_POLICY +from .redaction import redact, scan, verify_redaction +from .writer import CausalityError, LedgerWriter, RedactionError __all__ = [ "GENESIS", @@ -26,6 +30,13 @@ "Hlc", "LedgerWriter", "CausalityError", + "RedactionError", + "REDACTION_POLICY", + "redact", + "scan", + "verify_redaction", + "causal_parent", + "json_int", "canonicalize", "chain_hash", "verify_chain", diff --git a/adapters/korg-ledger-py/src/korg_ledger/__main__.py b/adapters/korg-ledger-py/src/korg_ledger/__main__.py new file mode 100644 index 0000000..5901b12 --- /dev/null +++ b/adapters/korg-ledger-py/src/korg_ledger/__main__.py @@ -0,0 +1,151 @@ +"""``python -m korg_ledger`` — import envelopes and lint journals. + + python -m korg_ledger envelope okto-session SESSION.json --source-version 0.1.0 > env.json + python -m korg_ledger envelope okto-genome genome.jsonl --source-version 0.1.0 > env.json + python -m korg_ledger envelope dazai-log daemon.log --source-version 0.1.0 > env.json + python -m korg_ledger import env.json --ledger .korg/ledger.jsonl [--triggered-by N] + python -m korg_ledger lint .korg/ledger.jsonl # chain + causal DAG + redaction + python -m korg_ledger rechain-redacted old.jsonl new.jsonl # migration: redacted copy, new chain + +Exit codes: 0 ok · 1 rejected / lint failure · 2 usage or parse error. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from ._hash import verify_chain, verify_dag +from .imports import ( + OKTO_PROFILE, + SCHEMA, + ImportRejected, + dazai_log_to_envelope, + import_envelope, + okto_mutation_record, + okto_session_to_envelope, +) +from .redaction import rechain_redacted, verify_redaction +from .writer import LedgerWriter + + +def _json_objects(text: str) -> list: + text = text.strip() + if not text: + return [] + if text.startswith("["): + return json.loads(text) + try: + return [json.loads(text)] # a single (possibly pretty-printed) object + except json.JSONDecodeError: + return [json.loads(line) for line in text.splitlines() if line.strip()] + + +def _cmd_envelope(a) -> int: + text = Path(a.file).read_text(encoding="utf-8") + if a.source == "okto-session": + objs = _json_objects(text) + if len(objs) != 1 or not isinstance(objs[0], dict): + print("expected exactly one Okto Session object", file=sys.stderr) + return 2 + env = okto_session_to_envelope(objs[0], okto_version=a.source_version) + elif a.source == "okto-genome": + env = { + "schema": SCHEMA, + "profile": OKTO_PROFILE, + "source": {"name": "okto", "version": a.source_version}, + "records": [okto_mutation_record(m) for m in _json_objects(text) if isinstance(m, dict)], + } + else: + env = dazai_log_to_envelope(text.splitlines(), version=a.source_version) + json.dump(env, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + return 0 + + +def _cmd_import(a) -> int: + env = json.loads(Path(a.envelope).read_text(encoding="utf-8")) + try: + seqs = import_envelope(LedgerWriter(a.ledger), env, triggered_by=a.triggered_by) + except ImportRejected as e: + print(f"import rejected ({len(e.errors)} problem(s)); nothing was written:", file=sys.stderr) + for err in e.errors[:50]: + print(f" - {err}", file=sys.stderr) + return 1 + print(f"imported {len(seqs) - 1} record(s) as seq {seqs[0]}..{seqs[-1]} (batch seq {seqs[0]})") + return 0 + + +def _cmd_lint(a) -> int: + events = _json_objects(Path(a.journal).read_text(encoding="utf-8")) + key = a.key.encode() if a.key else None + errors = verify_chain(events, key) + verify_dag(events) + verify_redaction(events) + if errors: + print(f"✗ {len(errors)} problem(s):") + for e in errors[:50]: + print(f" - {e}") + return 1 + print(f"✓ {len(events)} events: chain intact, causal DAG well-formed, redaction-clean (korg-redact@v1)") + return 0 + + +def _cmd_rechain(a) -> int: + events = _json_objects(Path(a.journal).read_text(encoding="utf-8")) + key = a.key.encode() if a.key else None + errors = verify_chain(events, key) + if errors and not a.force: + print("refusing to re-chain a journal that does not verify (pass --force to copy it anyway):") + for e in errors[:20]: + print(f" - {e}") + return 1 + out = rechain_redacted(events, key) + if Path(a.out).exists() and Path(a.out).stat().st_size: + print(f"refusing to overwrite non-empty {a.out}", file=sys.stderr) + return 2 + Path(a.out).write_text("".join(json.dumps(e, separators=(",", ":")) + "\n" for e in out), encoding="utf-8") + old_tip = events[-1].get("entry_hash") if events else None + new_tip = out[-1]["entry_hash"] if out else None + print(f"wrote {len(out)} redacted events to {a.out}") + print(f" old tip {old_tip}\n new tip {new_tip}\n record both tips (and anchor them) to keep the copy auditable") + return 0 + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="python -m korg_ledger", description=__doc__.split("\n\n")[0]) + sub = p.add_subparsers(dest="cmd", required=True) + + e = sub.add_parser("envelope", help="build a korg-import@v1 envelope from a sibling tool's native output") + e.add_argument("source", choices=["okto-session", "okto-genome", "dazai-log"]) + e.add_argument("file") + e.add_argument("--source-version", required=True, help="version of the tool that produced FILE") + e.set_defaults(fn=_cmd_envelope) + + i = sub.add_parser("import", help="validate an envelope and append it to a ledger (atomic on rejection)") + i.add_argument("envelope") + i.add_argument("--ledger", required=True) + i.add_argument("--triggered-by", type=int, default=None) + i.set_defaults(fn=_cmd_import) + + lint = sub.add_parser("lint", help="verify chain + causal DAG + redaction cleanliness of a journal") + lint.add_argument("journal") + lint.add_argument("--key", default=None, help="HMAC key for keyed chains") + lint.set_defaults(fn=_cmd_lint) + + r = sub.add_parser("rechain-redacted", help="write a redacted, re-chained copy of a journal (migration)") + r.add_argument("journal") + r.add_argument("out") + r.add_argument("--key", default=None, help="HMAC key: verifies the input and keys the new chain") + r.add_argument("--force", action="store_true", help="copy even if the input chain does not verify") + r.set_defaults(fn=_cmd_rechain) + + a = p.parse_args(argv) + try: + return a.fn(a) + except (OSError, json.JSONDecodeError) as ex: + print(f"error: {ex}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/adapters/korg-ledger-py/src/korg_ledger/_hash.py b/adapters/korg-ledger-py/src/korg_ledger/_hash.py index 4ee6d1b..4dc7a40 100644 --- a/adapters/korg-ledger-py/src/korg_ledger/_hash.py +++ b/adapters/korg-ledger-py/src/korg_ledger/_hash.py @@ -121,27 +121,84 @@ def verify_chain(events: list, key: bytes | None = None) -> list: return errors +def json_int(value): + """The integer a JSON value denotes, or None (korg-ledger@v1 §9.2). + + A ledger integer is a JSON number with an integral value within ±(2^53-1). + Booleans are NOT integers (Python's ``bool`` subclasses ``int``; Rust and JS + treat ``true`` as a boolean), and an integral float such as ``2.0`` IS the + integer 2 (JS cannot tell ``2.0`` from ``2`` after parsing, so all three + verifiers must accept it to agree).""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if abs(value) <= _SAFE_INT_MAX else None + if isinstance(value, float) and value.is_integer() and abs(value) <= _SAFE_INT_MAX: + return int(value) + return None + + +def _seq_label(e: dict) -> str: + sid = json_int(e.get("seq_id")) + return "?" if sid is None else str(sid) + + +def causal_parent(event: dict): + """Resolve an event's causal parent across the flat and nested shapes (§9.3). + + Returns ``(parent, error)``: ``parent`` is the parent seq_id (int) or None + for a root; ``error`` is a message (without the ``seq N:`` prefix) when the + link is malformed. A flat record carries ``triggered_by`` at the top level, a + nested JournalEvent under ``metadata``; ``null`` and absence both mean root. + When both are present they must name the same parent.""" + raw = [] + if event.get("triggered_by") is not None: + raw.append(("top-level", event["triggered_by"])) + md = event.get("metadata") + if isinstance(md, dict) and md.get("triggered_by") is not None: + raw.append(("metadata", md["triggered_by"])) + if not raw: + return None, None + parsed = [] + for _where, v in raw: + n = json_int(v) + if n is None: + return None, "triggered_by is not an integer" + parsed.append(n) + if len(parsed) == 2 and parsed[0] != parsed[1]: + return None, f"conflicting triggered_by (top-level {parsed[0]} vs metadata {parsed[1]})" + return parsed[0], None + + def verify_dag(events: list) -> list: - """Check the causal DAG: unique seq_ids and strictly-earlier triggered_by - links. Byte-for-byte equivalent to the JS ``verifyDag`` and the Rust - ``verify_dag``; ``triggered_by`` is read at the top level, so nested - JournalEvent records (which carry it under ``metadata``) get the - uniqueness check only — matching the other two implementations.""" + """Check the causal DAG (korg-ledger@v1 §5 + §9): unique integer seq_ids, and + every causal parent — top-level ``triggered_by`` (flat) or + ``metadata.triggered_by`` (nested JournalEvent) — is a well-formed integer + that references an existing, strictly-earlier seq_id. + + Error strings are byte-identical to the Rust ``verify_dag`` and the JS + ``verifyDag``; the conformance manifest pins them.""" errors: list[str] = [] if not isinstance(events, list): return ["events is not a list"] dicts = [e for e in events if isinstance(e, dict)] - seqs = [e.get("seq_id") for e in dicts if isinstance(e.get("seq_id"), int)] + seqs = [n for n in (json_int(e.get("seq_id")) for e in dicts) if n is not None] seqset = set(seqs) if len(seqset) != len(seqs): errors.append("duplicate seq_id present") for e in dicts: - tb = e.get("triggered_by") - if not isinstance(tb, int): + label = _seq_label(e) + tb, err = causal_parent(e) + if err is not None: + errors.append(f"seq {label}: {err}") continue - sid = e.get("seq_id") + if tb is None: + continue + sid = json_int(e.get("seq_id")) if tb not in seqset: - errors.append(f"seq {sid}: triggered_by {tb} does not exist") - elif isinstance(sid, int) and tb >= sid: - errors.append(f"seq {sid}: triggered_by {tb} is not strictly earlier") + errors.append(f"seq {label}: triggered_by {tb} does not exist") + elif sid is None: + errors.append(f"seq ?: triggered_by {tb} cannot be ordered (seq_id is not an integer)") + elif tb >= sid: + errors.append(f"seq {label}: triggered_by {tb} is not strictly earlier") return errors diff --git a/adapters/korg-ledger-py/src/korg_ledger/imports.py b/adapters/korg-ledger-py/src/korg_ledger/imports.py new file mode 100644 index 0000000..4201972 --- /dev/null +++ b/adapters/korg-ledger-py/src/korg_ledger/imports.py @@ -0,0 +1,500 @@ +"""korg-import@v1 — import envelopes for evidence korg did not observe itself. + +Normative text: spec/korg-ledger-v1/IMPORT.md. Two profiles ship: + +* ``okto.diagnostics@v1`` — sanitized diagnostics from the Okto agent-traffic + proxy (session/turn/anomaly/mutation/replay *metadata*; never headers, bodies, + raw stream bytes, or anomaly text). +* ``ningen-shikkaku.lifecycle@v1`` — lifecycle metadata from the + ningen-shikkaku dead-man's switch (armed/trigger/panic/wipe/pid events; never a + secret value, tool spec, argv, or free-text message). + +Safety is by **allowlist**, not by scrubbing: every record kind declares the only +fields it may carry and their types; anything else rejects the WHOLE envelope +before a single event is appended (fail closed, atomic). Strings are restricted +to identifiers and enums — there is no free-text field in either profile — and +the korg-redact@v1 scanner runs over every record as a second gate. + +The converters (``okto_session_to_envelope``, ``okto_mutation_record``, +``dazai_log_to_envelope``) build envelopes by *projection*: they read only the +allowlisted fields from the source's native output, so secret-bearing fields are +never even copied. +""" +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Iterable + +from ._events import agent_tool_call_event +from ._hash import canonicalize +from .redaction import scan + +SCHEMA = "korg-import@v1" +ACTOR_ID = "korg:import" + +_SAFE_INT_MAX = 2**53 - 1 +_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:\-]{0,127}") +_SLUG_RE = re.compile(r"[a-z0-9][a-z0-9_.\-]{0,63}") +_VERSION_RE = re.compile(r"[0-9A-Za-z][0-9A-Za-z.+\-]{0,31}") + + +class ImportRejected(ValueError): + """The envelope violates korg-import@v1; nothing was appended.""" + + def __init__(self, errors: list[str]): + super().__init__("; ".join(errors)) + self.errors = errors + + +# ── field types ────────────────────────────────────────────────────────────── + + +def _nonneg_int(v, _): + return isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= _SAFE_INT_MAX + + +def _bool(v, _): + return isinstance(v, bool) + + +def _id(v, _): + return isinstance(v, str) and _ID_RE.fullmatch(v) is not None + + +def _slug(v, _): + return isinstance(v, str) and _SLUG_RE.fullmatch(v) is not None + + +def _enum(*allowed): + def check(v, _): + return isinstance(v, str) and v in allowed + + check.allowed = allowed # type: ignore[attr-defined] + return check + + +def _int_range(lo, hi): + def check(v, _): + return isinstance(v, int) and not isinstance(v, bool) and lo <= v <= hi + + return check + + +def _slug_count_map(v, _): + return ( + isinstance(v, dict) + and len(v) <= 16 + and all(_slug(k, None) and _nonneg_int(n, None) for k, n in v.items()) + ) + + +# ── profiles ───────────────────────────────────────────────────────────────── +# kind → (required fields, optional fields); each field → type check. + +_OKTO_CLIENTS = _enum( + "claude", "gemini", "codex", "agy", "kimi", "kimi-code", "cursor", "opencode", "pi", "hermes", "unknown" +) +_OKTO_ANOMALY = _enum( + "bare_json_without_data_prefix", + "malformed_json_in_sse_frame", + "empty_response_body", + "line_ending_mismatch", + "protocol_anomaly", + "other", +) + +OKTO_PROFILE = "okto.diagnostics@v1" +NINGEN_PROFILE = "ningen-shikkaku.lifecycle@v1" + +_AT = {"at_ms": _nonneg_int} + +PROFILES: dict[str, dict[str, Any]] = { + OKTO_PROFILE: { + "source": "okto", + "kinds": { + "session": ( + {"session_id": _id, "client": _OKTO_CLIENTS}, + {**_AT, "tls_established": _bool, "synthetic": _bool, "provider_vendor": _slug, + "provider_product": _slug, "turn_count": _nonneg_int, "anomaly_count": _nonneg_int, + "mutation_count": _nonneg_int}, + ), + "turn": ( + {"session_id": _id, "turn_index": _nonneg_int}, + {**_AT, "status_code": _int_range(100, 599), "duration_ms": _nonneg_int, + "chunk_count": _nonneg_int, "anomaly_count": _nonneg_int, + "mutations_applied": _nonneg_int, "token_usage": _slug_count_map}, + ), + "anomaly": ( + {"session_id": _id, "turn_index": _nonneg_int, "chunk_index": _nonneg_int, + "anomaly_type": _OKTO_ANOMALY}, + {**_AT, "repaired": _bool}, + ), + "mutation": ( + {"mutation_id": _id, "scope": _enum("request_header", "request_body", "response_header", + "response_body", "stream_chunk"), + "patch_type": _enum("literal", "regex_sub", "json_patch", "header_set")}, + {**_AT, "session_id": _id, "client": _OKTO_CLIENTS, "confidence_bp": _int_range(0, 10000), + "reinforcement_count": _nonneg_int}, + ), + "replay": ( + {"session_id": _id, "resolved": _bool}, + {**_AT, "total_anomalies": _nonneg_int, "total_mutations_applied": _nonneg_int}, + ), + }, + # Named so a rejection explains *why* (these carry secrets or raw traffic). + "forbidden": ( + "request_headers", "response_headers", "headers", "request_body", "response_body", "body", + "raw_hex", "expected_hex", "patched_hex", "pattern_hex", "patch_hex", "description", + "parsed", "chunks", "stream_chunks", "path", "url", "connect_target", "hostname", "node_id", + ), + }, + NINGEN_PROFILE: { + "source": "ningen-shikkaku", + "kinds": { + "daemon_start": ( + {"mode": _enum("armed", "dry_run")}, + {**_AT, "pid": _nonneg_int, "grace_ms": _nonneg_int, "ping_timeout_ms": _nonneg_int, + "seccomp": _bool}, + ), + "buffers": ({"allocated": _nonneg_int}, {**_AT, "locked": _nonneg_int}), + "status": ( + {"alive": _bool, "armed": _bool}, + {**_AT, "grace_seconds": _nonneg_int, "registered_pids": _nonneg_int}, + ), + "armed": ({"via": _enum("flag", "control", "mcp")}, dict(_AT)), + "trigger": ( + {"reason": _enum("heartbeat_dropped", "ping_deadline_missed", "sigusr1", "sigusr2", + "grace_expired", "other"), + "mode": _enum("armed", "dry_run")}, + {**_AT, "severity": _enum("graceful", "hard")}, + ), + "cancelled": ({"reason": _enum("client_reconnected", "client_cancel", "other")}, dict(_AT)), + "panic": ( + {"reason": _enum("heartbeat_dropped", "ping_deadline_missed", "sigusr1", "sigusr2", + "grace_expired", "other")}, + dict(_AT), + ), + "wiped": ({"buffers": _nonneg_int}, dict(_AT)), + "pid_registered": ({"pid": _nonneg_int}, {**_AT, "total": _nonneg_int}), + "pid_unregistered": ({"pid": _nonneg_int}, dict(_AT)), + "pid_killed": ({"pid": _nonneg_int, "outcome": _enum("sent", "already_gone")}, dict(_AT)), + "shutdown": ({"how": _enum("quit", "signal")}, dict(_AT)), + "exit_condition_fired": ( + {"counter_mode": _enum("calls", "session", "either")}, + {**_AT, "calls": _nonneg_int}, + ), + }, + "forbidden": ( + "value", "secret", "secret_value", "plaintext", "data", "bytes", "cmd", "command", "spec", + "tool_spec", "stdout", "stderr", "output", "argv", "args", "env", "message", "reply", + "buffer_address", "address", + ), + }, +} + +_ENVELOPE_FIELDS = {"schema", "profile", "source", "records", "exported_at_ms", "dropped"} + + +def validate_envelope(env: Any) -> list[str]: + """All korg-import@v1 violations in ``env`` (``[]`` iff importable).""" + if not isinstance(env, dict): + return ["envelope is not a JSON object"] + errors: list[str] = [] + for k in sorted(set(env) - _ENVELOPE_FIELDS): + errors.append(f"envelope: unknown field {k!r}") + if env.get("schema") != SCHEMA: + errors.append(f"envelope: schema must be {SCHEMA!r}") + profile = PROFILES.get(env.get("profile")) + if profile is None: + errors.append(f"envelope: unknown profile (known: {sorted(PROFILES)})") + return errors + src = env.get("source") + if not (isinstance(src, dict) and set(src) == {"name", "version"}): + errors.append("envelope: source must be exactly {name, version}") + else: + if src["name"] != profile["source"]: + errors.append(f"envelope: source.name must be {profile['source']!r} for {env['profile']}") + if not (isinstance(src["version"], str) and _VERSION_RE.fullmatch(src["version"])): + errors.append("envelope: source.version must match [0-9A-Za-z][0-9A-Za-z.+-]{0,31}") + for k in ("exported_at_ms", "dropped"): + if k in env and not _nonneg_int(env[k], None): + errors.append(f"envelope: {k} must be a non-negative integer") + records = env.get("records") + if not isinstance(records, list) or not records: + errors.append("envelope: records must be a non-empty array") + return errors + if len(records) > 10_000: + errors.append("envelope: more than 10000 records") + return errors + kinds, forbidden = profile["kinds"], profile["forbidden"] + for i, rec in enumerate(records): + where = f"records[{i}]" + if not isinstance(rec, dict): + errors.append(f"{where}: not an object") + continue + kind = rec.get("kind") + if kind not in kinds: + errors.append(f"{where}: unknown kind (known: {sorted(kinds)})") + continue + required, optional = kinds[kind] + for f in sorted(set(rec) - {"kind"}): + if f in forbidden: + errors.append(f"{where}: forbidden field {f!r} (may carry secrets or raw traffic)") + elif f not in required and f not in optional: + errors.append(f"{where}: field {f!r} is not allowed for kind {kind!r}") + elif not (required.get(f) or optional[f])(rec[f], f): + allowed = getattr(required.get(f) or optional[f], "allowed", None) + hint = f" (one of {list(allowed)})" if allowed else "" + errors.append(f"{where}: field {f!r} has an invalid value{hint}") + for f in sorted(set(required) - set(rec)): + errors.append(f"{where}: missing required field {f!r}") + for finding in scan(rec): + errors.append(f"{where}: secret-shaped value at {finding['path']} ({finding['rule']})") + return errors + + +def envelope_digest(env: dict) -> str: + """sha256 over the canonical envelope bytes — lets a holder of the original + sanitized envelope prove it is exactly what was imported.""" + return hashlib.sha256(canonicalize(env)).hexdigest() + + +def import_envelope(writer, env: dict, *, triggered_by: int | None = None, actor_id: str = ACTOR_ID) -> list[int]: + """Validate, then append one ``korg.import`` batch event plus one event per + record (each ``triggered_by`` the batch). Returns the appended seq_ids. + + Atomic with respect to validation: an invalid envelope raises + ``ImportRejected`` before anything is written.""" + errors = validate_envelope(env) + if errors: + raise ImportRejected(errors) + src = env["source"] + agent = f"import:{src['name']}@{src['version']}" + digest = envelope_digest(env) + batch_args = { + "profile": env["profile"], + "record_count": len(env["records"]), + "envelope_sha256": digest, + } + for k in ("exported_at_ms", "dropped"): + if k in env: + batch_args[k] = env[k] + seqs = [ + writer.append( + event=agent_tool_call_event( + source_agent=agent, tool_name="korg.import", args=batch_args, result={}, success=True, duration_ms=0 + ), + actor_id=actor_id, + triggered_by=triggered_by, + ) + ] + for i, rec in enumerate(env["records"]): + args = {k: v for k, v in rec.items() if k != "kind"} + seqs.append( + writer.append( + event=agent_tool_call_event( + source_agent=agent, + tool_name=f"{src['name']}.{rec['kind']}", + args=args, + result={"import_index": i}, + success=True, + duration_ms=0, + ), + actor_id=actor_id, + triggered_by=seqs[0], + ) + ) + return seqs + + +# ── converters: native sibling output → envelope (projection only) ────────── + + +def _slugify(s: Any) -> str | None: + if not isinstance(s, str): + return None + out = re.sub(r"[^a-z0-9_.\-]+", "-", s.lower()).strip("-")[:64] + return out if out and _SLUG_RE.fullmatch(out) else None + + +def _ms(v: Any) -> int | None: + if isinstance(v, bool) or not isinstance(v, (int, float)) or v != v or v < 0: + return None + n = int(round(v)) + return n if n <= _SAFE_INT_MAX else None + + +def okto_session_to_envelope(session: dict, *, okto_version: str) -> dict: + """Project an Okto ``Session`` (``~/.okto/sessions/.jsonl``) onto + ``okto.diagnostics@v1``. Reads ONLY: id, client, started_at_hlc.wall_ms, + tls_established, synthetic, provider.{vendor,product}, and per turn: index, + response_status, duration_ms, token_usage, anomalies_detected, + mutations_applied (counted), and each stream chunk's index/anomalous/ + anomaly_type/applied_mutation_id. Headers, bodies, raw_hex, parsed chunks, + connect_target and hostname are never read.""" + sid = session.get("id") + client = session.get("client") if session.get("client") in _OKTO_CLIENTS.allowed else "unknown" + rec: dict[str, Any] = {"kind": "session", "session_id": sid, "client": client} + started = _ms((session.get("started_at_hlc") or {}).get("wall_ms")) + if started is not None: + rec["at_ms"] = started + for k in ("tls_established", "synthetic"): + if isinstance(session.get(k), bool): + rec[k] = session[k] + provider = session.get("provider") or {} + for src_k, dst_k in (("vendor", "provider_vendor"), ("product", "provider_product")): + slug = _slugify(provider.get(src_k)) if isinstance(provider, dict) else None + if slug: + rec[dst_k] = slug + records = [rec] + dropped = 0 + turns = session.get("turns") or [] + total_anomalies = total_mutations = 0 + for t in turns: + if not isinstance(t, dict): + dropped += 1 + continue + idx = t.get("index") + chunks = t.get("stream_chunks") or [] + mutations = t.get("mutations_applied") or [] + turn = {"kind": "turn", "session_id": sid, "turn_index": idx, "chunk_count": len(chunks), + "mutations_applied": len(mutations) if isinstance(mutations, list) else 0} + status = t.get("response_status") + if isinstance(status, int) and not isinstance(status, bool) and 100 <= status <= 599: + turn["status_code"] = status + dur = _ms(t.get("duration_ms")) + if dur is not None: + turn["duration_ms"] = dur + usage = t.get("token_usage") + if isinstance(usage, dict): + clean = {_slugify(k): v for k, v in usage.items() if _slugify(k) and _nonneg_int(v, None)} + if clean: + turn["token_usage"] = dict(list(sorted(clean.items()))[:16]) + anomalies = 0 + chunk_records = [] + for c in chunks: + if isinstance(c, dict) and c.get("anomalous") is True: + anomalies += 1 + atype = c.get("anomaly_type") + chunk_records.append({ + "kind": "anomaly", "session_id": sid, "turn_index": idx, "chunk_index": c.get("index"), + "anomaly_type": atype if atype in _OKTO_ANOMALY.allowed else "other", + "repaired": c.get("applied_mutation_id") is not None, + }) + turn["anomaly_count"] = anomalies + total_anomalies += anomalies + total_mutations += turn["mutations_applied"] + records.append(turn) + records.extend(chunk_records) + rec["turn_count"] = len(turns) + rec["anomaly_count"] = total_anomalies + rec["mutation_count"] = total_mutations + env = {"schema": SCHEMA, "profile": OKTO_PROFILE, "source": {"name": "okto", "version": okto_version}, + "records": records} + if dropped: + env["dropped"] = dropped + return env + + +def okto_mutation_record(mutation: dict) -> dict: + """Project one Okto genome ``Mutation`` (``~/.okto/genome.jsonl``) onto a + ``mutation`` record. ``pattern_hex``, ``patch_hex`` and ``description`` (which + embed real traffic bytes) are never read; ``confidence`` becomes integer + basis points because ledger numbers are integers.""" + rec: dict[str, Any] = { + "kind": "mutation", + "mutation_id": mutation.get("id"), + "scope": mutation.get("scope"), + "patch_type": mutation.get("patch_type"), + } + at = _ms((mutation.get("hlc") or {}).get("wall_ms")) + if at is not None: + rec["at_ms"] = at + if isinstance(mutation.get("session_id"), str): + rec["session_id"] = mutation["session_id"] + if mutation.get("client") in _OKTO_CLIENTS.allowed: + rec["client"] = mutation["client"] + conf = mutation.get("confidence") + if isinstance(conf, (int, float)) and not isinstance(conf, bool) and 0 <= conf <= 1: + rec["confidence_bp"] = int(round(conf * 10000)) + rc = mutation.get("reinforcement_count") + if _nonneg_int(rc, None): + rec["reinforcement_count"] = rc + return rec + + +_DAZAI_REASONS = { + "client reconnected": "client_reconnected", + "client CANCEL": "client_cancel", + "heartbeat connection dropped": "heartbeat_dropped", + "ping deadline missed": "ping_deadline_missed", + "SIGUSR1 panic signal": "sigusr1", + "SIGUSR2 hard panic": "sigusr2", + "grace window expired": "grace_expired", +} + +_TRIGGER_REASONS = {"heartbeat_dropped", "ping_deadline_missed", "sigusr1", "sigusr2", "grace_expired"} + + +def _reason(text: str, allowed: set[str]) -> str: + r = _DAZAI_REASONS.get(text.strip(), "other") + return r if r in allowed else "other" + + +# (pattern, builder) — matched with re.search so both the Rust "[dazai] …" and +# the Python reference "HH:MM:SS [dazai.deadman] …" log formats are accepted. +_DAZAI_LINES: list[tuple[re.Pattern[str], Any]] = [ + (re.compile(r"watchdog pid=(\d+) mode=(ARMED|DRY-RUN)"), + lambda m: {"kind": "daemon_start", "pid": int(m[1]), "mode": "armed" if m[2] == "ARMED" else "dry_run"}), + (re.compile(r"allocated (\d+) secret buffer\(s\), (\d+) mlock'd"), + lambda m: {"kind": "buffers", "allocated": int(m[1]), "locked": int(m[2])}), + (re.compile(r"ARMED at runtime via control message"), lambda m: {"kind": "armed", "via": "control"}), + (re.compile(r"DRY-RUN (HARD )?trigger \((.+?)\)"), + lambda m: {"kind": "trigger", "mode": "dry_run", "severity": "hard" if m[1] else "graceful", + "reason": _reason(m[2], _TRIGGER_REASONS)}), + (re.compile(r"ARMED trigger \((.+?)\)"), + lambda m: {"kind": "trigger", "mode": "armed", "severity": "graceful", + "reason": _reason(m[1], _TRIGGER_REASONS)}), + (re.compile(r"panic CANCELLED \((.+?)\)"), + lambda m: {"kind": "cancelled", "reason": _reason(m[1], {"client_reconnected", "client_cancel"})}), + (re.compile(r"PANIC \((.+?)\)"), lambda m: {"kind": "panic", "reason": _reason(m[1], _TRIGGER_REASONS)}), + (re.compile(r"SIGKILL registered pid (\d+): (sent|already gone)"), + lambda m: {"kind": "pid_killed", "pid": int(m[1]), "outcome": m[2].replace(" ", "_")}), + (re.compile(r"wiped (\d+) secret buffer\(s\)"), lambda m: {"kind": "wiped", "buffers": int(m[1])}), + (re.compile(r"zeroized (\d+) buffer\(s\)"), lambda m: {"kind": "wiped", "buffers": int(m[1])}), + (re.compile(r"registered pid (\d+) \((\d+) total\)"), + lambda m: {"kind": "pid_registered", "pid": int(m[1]), "total": int(m[2])}), + (re.compile(r"client QUIT: intentional clean stand-down"), lambda m: {"kind": "shutdown", "how": "quit"}), + (re.compile(r"SIGTERM/SIGINT: clean shutdown"), lambda m: {"kind": "shutdown", "how": "signal"}), +] + + +def dazai_log_to_envelope(lines: Iterable[str], *, version: str) -> dict: + """Project ningen-shikkaku daemon stderr (``[dazai] …`` lines) onto + ``ningen-shikkaku.lifecycle@v1``. Only the fixed lifecycle lines are + recognised; their numeric/enum parts are extracted and everything else on the + line is discarded. Unrecognised lines (including any ``error: …`` text, which + can echo a tool spec) are dropped and counted in ``dropped``.""" + records: list[dict] = [] + dropped = 0 + for line in lines: + if not line.strip(): + continue + for pat, build in _DAZAI_LINES: + m = pat.search(line) + if m: + records.append(build(m)) + break + else: + dropped += 1 + env = {"schema": SCHEMA, "profile": NINGEN_PROFILE, + "source": {"name": "ningen-shikkaku", "version": version}, "records": records} + if dropped: + env["dropped"] = dropped + return env + + +def load_envelope(text: str) -> dict: + return json.loads(text) diff --git a/adapters/korg-ledger-py/src/korg_ledger/redaction.py b/adapters/korg-ledger-py/src/korg_ledger/redaction.py new file mode 100644 index 0000000..3ca1c16 --- /dev/null +++ b/adapters/korg-ledger-py/src/korg_ledger/redaction.py @@ -0,0 +1,332 @@ +"""korg-redact@v1 — redaction at persistence (stdlib only). + +Normative text: spec/korg-ledger-v1/REDACTION.md. Byte-for-byte equivalent to +the Rust ``korg_ledger::redact`` module; both are pinned by the shared cases in +spec/korg-ledger-v1/vectors/redaction/korg-redact-v1.json. + +A chained event cannot be redacted after it is hashed (that is tampering), so a +producer MUST run ``redact`` on the record *before* ``chain_hash`` and before +anything touches disk. The policy is deterministic and idempotent: +``redact(redact(x)) == redact(x)``, and a value is redaction-clean iff +``scan(value) == []``. + +Two rule families: + +* **sensitive keys** — an object member whose normalized key is a credential + name (``authorization``, ``api_key``, ``client_secret``, ``*_token`` …) has its + whole value replaced by ``[REDACTED:sensitive_key]`` (null, booleans and "" are + kept: they carry no secret). +* **value patterns** — every string is scanned for credential shapes (PEM private + keys, URL userinfo, bearer tokens, provider API keys, JWTs, ``NAME_TOKEN=…`` + assignments, ``"password": "…"`` JSON fields) and the secret part is replaced + in place by ``[REDACTED:]``. + +Markers never carry any function of the secret (no hash, prefix, or length), so +a persisted ledger gives no oracle for guessing low-entropy secrets. + +The patterns deliberately use ASCII classes only (no ``\\s``, ``\\b``, ``\\w`` or +case-insensitive flags): Python ``re`` and the Rust ``regex`` crate disagree on +their Unicode semantics, and a split would make one implementation persist what +the other would redact. +""" +from __future__ import annotations + +import re +from typing import Any, Callable + +from ._hash import json_int + +#: Policy identifier recorded on every redacted event (``metadata.tags`` on a +#: nested JournalEvent, top-level ``redaction`` on a flat record). +POLICY = "korg-redact@v1" + +#: Tag key under ``metadata.tags`` that declares the policy a producer applied. +TAG = "korg.redaction" + + +def marker(rule: str) -> str: + return f"[REDACTED:{rule}]" + + +_MARKER_RE = re.compile(r"\[REDACTED:[a-z_]+\]") + +#: Exact normalized key names whose values are credentials. +SENSITIVE_KEYS = frozenset( + { + "authorization", + "proxy_authorization", + "cookie", + "set_cookie", + "api_key", + "apikey", + "password", + "passwd", + "passphrase", + "secret", + "token", + "private_key", + "secret_key", + "aws_secret_access_key", + "credential", + "credentials", + } +) + +#: Normalized key suffixes whose values are credentials (``openai_api_key``, +#: ``client_secret``, ``access_token``, ``x_amz_security_token`` …). Note +#: ``prompt_tokens`` / ``max_tokens`` do NOT match (plural). +SENSITIVE_KEY_SUFFIXES = ( + "_api_key", + "_apikey", + "_secret", + "_secret_key", + "_private_key", + "_password", + "_passwd", + "_passphrase", + "_token", +) + + +def normalize_key(key: str) -> str: + """camelCase / kebab-case / spaced → snake_case, ASCII-lowercased. + + ``accessToken`` → ``access_token``, ``X-Api-Key`` → ``x_api_key``, + ``APIKey`` → ``apikey``. Only ASCII letters change case.""" + out = [] + prev = "" + for c in key: + if "A" <= c <= "Z": + if prev and ("a" <= prev <= "z" or "0" <= prev <= "9"): + out.append("_") + out.append(chr(ord(c) + 32)) + elif c in "- ": + out.append("_") + else: + out.append(c) + prev = c + return "".join(out) + + +def is_sensitive_key(key: str) -> bool: + k = normalize_key(key) + return k in SENSITIVE_KEYS or k.endswith(SENSITIVE_KEY_SUFFIXES) + + +# Leading "boundary": start of string or a non-[A-Za-z0-9_] character, captured +# and re-emitted (Rust regex has no look-behind). +_B = r"(^|[^A-Za-z0-9_])" + +_SECRET_JSON_KEYS = ( + "password|passwd|passphrase|secret|client_secret|api_key|apikey|" + "access_token|refresh_token|auth_token|id_token|token|private_key|authorization" +) + +#: Ordered (rule_id, compiled pattern, replacement). Order is normative. +_RULES: list[tuple[str, "re.Pattern[str]", Callable[["re.Match[str]"], str]]] = [ + ( + "private_key_block", + re.compile( + r"-----BEGIN[A-Z ]*PRIVATE KEY-----" + r"(?:[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----|[\s\S]*)" + ), + lambda m: marker("private_key_block"), + ), + ( + "url_credentials", + re.compile(r"([A-Za-z][A-Za-z0-9+.\-]*://)[^ \t\r\n/:@\[\]]+:[^ \t\r\n/@\[\]]+@"), + lambda m: m.group(1) + marker("url_credentials") + "@", + ), + ( + "bearer_token", + re.compile(r"((?:Bearer|bearer|BEARER)[ \t]+)[A-Za-z0-9._~+/\-]{16,}=*"), + lambda m: m.group(1) + marker("bearer_token"), + ), + ( + "provider_api_key", + re.compile(_B + r"sk-[A-Za-z0-9_\-]{20,}"), + lambda m: m.group(1) + marker("provider_api_key"), + ), + ( + "github_token", + re.compile(_B + r"(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{22,})"), + lambda m: m.group(1) + marker("github_token"), + ), + ( + "slack_token", + re.compile(_B + r"xox[abposr]-[A-Za-z0-9\-]{10,}"), + lambda m: m.group(1) + marker("slack_token"), + ), + ( + "aws_access_key_id", + re.compile(_B + r"(?:AKIA|ASIA)[0-9A-Z]{16}"), + lambda m: m.group(1) + marker("aws_access_key_id"), + ), + ( + "google_api_key", + re.compile(_B + r"AIza[0-9A-Za-z_\-]{35}"), + lambda m: m.group(1) + marker("google_api_key"), + ), + ( + "jwt", + re.compile(_B + r"eyJ[A-Za-z0-9_\-]{8,}\.eyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}"), + lambda m: m.group(1) + marker("jwt"), + ), + ( + "env_assignment", + re.compile( + _B + + r"([A-Z][A-Z0-9_]*(?:API_KEY|APIKEY|SECRET|SECRET_KEY|PASSWORD|PASSWD|TOKEN|PRIVATE_KEY|ACCESS_KEY))" + r"([ \t]*[=:][ \t]*)" + r"(?:\"[^\"\n]*\"|'[^'\n]*'|[^ \t\r\n\"',;&]+)" + ), + lambda m: m.group(1) + m.group(2) + m.group(3) + marker("env_assignment"), + ), + ( + "json_secret_field", + re.compile(r"(\"(?:" + _SECRET_JSON_KEYS + r")\"[ \t]*:[ \t]*)\"(?:[^\"\\\n]|\\.)*\""), + lambda m: m.group(1) + '"' + marker("json_secret_field") + '"', + ), +] + +#: Rule ids in normative order (also the key-rule id). +RULE_IDS = ("sensitive_key",) + tuple(r[0] for r in _RULES) + + +#: Upper bound on rule-list passes. Real input reaches its fixed point in one or +#: two passes; hitting the bound leaves the string unclean, which the writer's +#: fail-closed gate (R4) then refuses to persist. +MAX_PASSES = 8 + + +def redact_string(s: str) -> tuple[str, list[str]]: + """Apply the value-pattern rules, in order, repeatedly until the string stops + changing (a fixed point — so redaction is idempotent by construction even + where one replacement exposes a new match, e.g. two keys back to back). + Returns (new, ids of the rules that fired, in normative order).""" + fired: set[str] = set() + for _ in range(MAX_PASSES): + changed = False + for rule_id, pat, repl in _RULES: + new = pat.sub(repl, s) + if new != s: + fired.add(rule_id) + s = new + changed = True + if not changed: + break + return s, [r for r in RULE_IDS if r in fired] + + +def _pointer_escape(token: str) -> str: + return token.replace("~", "~0").replace("/", "~1") + + +def _redact(value: Any, path: str, findings: list[dict]) -> Any: + if isinstance(value, str): + new, fired = redact_string(value) + for r in fired: + findings.append({"path": path, "rule": r}) + return new + if isinstance(value, list): + return [_redact(v, f"{path}/{i}", findings) for i, v in enumerate(value)] + if isinstance(value, dict): + out = {} + for k, v in value.items(): + p = f"{path}/{_pointer_escape(k)}" + if is_sensitive_key(k) and not _keeps_under_sensitive_key(v): + findings.append({"path": p, "rule": "sensitive_key"}) + out[k] = marker("sensitive_key") + else: + out[k] = _redact(v, p, findings) + return out + return value + + +def _keeps_under_sensitive_key(v: Any) -> bool: + # null / booleans / "" carry no secret; an existing marker is already redacted. + if v is None or isinstance(v, bool) or v == "": + return True + return isinstance(v, str) and _MARKER_RE.fullmatch(v) is not None + + +def redact(value: Any) -> tuple[Any, list[dict]]: + """Redact a JSON value under korg-redact@v1. + + Returns ``(redacted_copy, findings)`` where each finding is + ``{"path": , "rule": }``. The input is not + mutated. Object keys are never rewritten (they are structure, not data).""" + findings: list[dict] = [] + return _redact(value, "", findings), findings + + +def scan(value: Any) -> list[dict]: + """Findings that ``redact`` would produce — empty iff ``value`` is clean.""" + return redact(value)[1] + + +#: Fields that are chain structure, never payload: excluded from redaction and +#: from the cleanliness lint (they are hex digests / integers by construction). +STRUCTURAL_FIELDS = ("prev_hash", "entry_hash", "event_sig") + + +def finding_order(f: dict) -> tuple: + """Canonical finding order: JSON pointer (code point), then normative rule + order. Implementations iterate object members in different orders, so every + reported list is sorted by this key.""" + return (f["path"], RULE_IDS.index(f["rule"])) + + +def scan_record(record: dict) -> list[dict]: + """Lint one persisted ledger record (flat or nested) for unredacted secrets.""" + body = {k: v for k, v in record.items() if k not in STRUCTURAL_FIELDS} + return scan(body) + + +def verify_redaction(events: list) -> list[str]: + """Journal-level lint: one error per finding, localized to ``seq N`` and a + JSON pointer. ``[]`` iff every event is redaction-clean under korg-redact@v1.""" + errors: list[str] = [] + if not isinstance(events, list): + return ["events is not a list"] + for e in events: + if not isinstance(e, dict): + continue + sid = json_int(e.get("seq_id")) + label = "?" if sid is None else str(sid) + for f in sorted(scan_record(e), key=finding_order): + errors.append(f"seq {label}: unredacted {f['rule']} at {f['path'] or '/'}") + return errors + + +def rechain_redacted(events: list, key: bytes | None = None) -> list: + """Migration helper (REDACTION.md R6): a NEW journal holding a redacted copy + of ``events``, re-chained from genesis. + + Each record body is redacted and declares the policy (``metadata.tags`` on a + nested record, top-level ``redaction`` on a flat one); ``seq_id`` and causal + links are preserved; ``prev_hash``/``entry_hash`` are recomputed (HMAC when + ``key`` is given). ``event_sig`` is dropped — the old signature covered the + unredacted bytes, so re-sign if you need per-event attribution. The result is + a different chain: record the old tip next to the new one (and anchor both) + so the provenance of the copy is auditable.""" + from ._hash import GENESIS, chain_hash # local: avoid widening module imports + + out: list = [] + prev = GENESIS + for e in events: + if not isinstance(e, dict): + raise ValueError("rechain_redacted: every event must be a JSON object") + body = {k: v for k, v in e.items() if k not in STRUCTURAL_FIELDS} + md = body.get("metadata") + if isinstance(md, dict) and isinstance(body.get("event"), dict): + tags = md.get("tags") if isinstance(md.get("tags"), dict) else {} + body["metadata"] = {**md, "tags": {**tags, TAG: POLICY}} + else: + body["redaction"] = POLICY + rec, _ = redact(body) + rec["prev_hash"] = prev + rec["entry_hash"] = chain_hash(rec, key) + prev = rec["entry_hash"] + out.append(rec) + return out diff --git a/adapters/korg-ledger-py/src/korg_ledger/writer.py b/adapters/korg-ledger-py/src/korg_ledger/writer.py index 105c8cd..f1f9db9 100644 --- a/adapters/korg-ledger-py/src/korg_ledger/writer.py +++ b/adapters/korg-ledger-py/src/korg_ledger/writer.py @@ -22,6 +22,9 @@ from ._events import NIL_UUID from ._hash import GENESIS, chain_hash from ._hlc import Hlc +from .redaction import POLICY as REDACTION_POLICY +from .redaction import TAG as REDACTION_TAG +from .redaction import redact, scan SCHEMA_VERSION = "1.0" @@ -30,6 +33,12 @@ class CausalityError(ValueError): """Raised when triggered_by does not reference a strictly-earlier seq_id.""" +class RedactionError(ValueError): + """Raised (fail closed) when a record is still not redaction-clean after the + policy ran. Unreachable while the policy is idempotent; kept as the gate that + guarantees nothing unclean is ever hashed or written.""" + + class LedgerWriter: def __init__( self, @@ -37,6 +46,7 @@ def __init__( hmac_key: bytes | None = None, hlc_actor_id: int = 1, signing_key: bytes | None = None, + redaction: str | None = REDACTION_POLICY, ) -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) @@ -47,6 +57,13 @@ def __init__( # signed (requires the `signing` extra / `cryptography`). Default None # keeps the writer stdlib-only. self._signing_key = signing_key + # Redaction at persistence (spec REDACTION.md): every record is redacted + # BEFORE it is hashed, signed, or written. ``None`` opts out explicitly + # (e.g. replaying a fixture byte-for-byte); the record then carries no + # redaction tag, so a reader can tell it was never redacted. + if redaction not in (REDACTION_POLICY, None): + raise ValueError(f"unsupported redaction policy {redaction!r} (want {REDACTION_POLICY!r} or None)") + self._redaction = redaction with self.path.open("r") as f: self._last_seq, self._last_hash, self._last_hlc = self._read_tip(f) @@ -101,6 +118,10 @@ def append( try: last_seq, last_hash, last_hlc = self._read_tip(f) seq_id = last_seq + 1 + if triggered_by is not None and ( + isinstance(triggered_by, bool) or not isinstance(triggered_by, int) + ): + raise CausalityError(f"triggered_by must be an integer seq_id, got {triggered_by!r}") if triggered_by is not None and (triggered_by < 1 or triggered_by >= seq_id): raise CausalityError( f"triggered_by {triggered_by} is not a strictly-earlier seq_id (< {seq_id})" @@ -127,8 +148,13 @@ def append( "seq_id": seq_id, "metadata": metadata, "event": event, - "prev_hash": last_hash, } + if self._redaction is not None: + metadata["tags"] = {**metadata["tags"], REDACTION_TAG: self._redaction} + record, _findings = redact(record) + if scan(record): + raise RedactionError(f"seq {seq_id}: record is not redaction-clean after {self._redaction}") + record["prev_hash"] = last_hash record["entry_hash"] = chain_hash(record, self._key) if self._signing_key is not None: from . import signing # lazy: keeps the stdlib path import-clean diff --git a/adapters/korg-ledger-py/tests/test_causal.py b/adapters/korg-ledger-py/tests/test_causal.py new file mode 100644 index 0000000..da5a431 --- /dev/null +++ b/adapters/korg-ledger-py/tests/test_causal.py @@ -0,0 +1,76 @@ +"""korg-ledger@v1 §9: causal-parent resolution across flat and nested shapes.""" +import json + +import pytest + +from korg_ledger import CausalityError, LedgerWriter, agent_tool_call_event, causal_parent, verify_dag + + +def test_nested_links_are_checked_not_skipped(): + events = [ + {"seq_id": 1, "metadata": {"triggered_by": None}}, + {"seq_id": 2, "metadata": {"triggered_by": 3}}, # forward reference + {"seq_id": 3, "metadata": {"triggered_by": 7}}, # dangling + ] + assert verify_dag(events) == [ + "seq 2: triggered_by 3 is not strictly earlier", + "seq 3: triggered_by 7 does not exist", + ] + + +@pytest.mark.parametrize( + "value, parent, error", + [ + (None, None, None), + (1, 1, None), + (1.0, 1, None), # JS cannot tell 1.0 from 1, so every impl accepts it + (True, None, "triggered_by is not an integer"), # bool is not an int (Python quirk) + (1.5, None, "triggered_by is not an integer"), + ("1", None, "triggered_by is not an integer"), + (2**53, None, "triggered_by is not an integer"), + ([1], None, "triggered_by is not an integer"), + ], +) +def test_integer_domain(value, parent, error): + assert causal_parent({"triggered_by": value}) == (parent, error) + assert causal_parent({"metadata": {"triggered_by": value}}) == (parent, error) + + +def test_both_shapes_must_agree(): + assert causal_parent({"triggered_by": 2, "metadata": {"triggered_by": 2}}) == (2, None) + assert causal_parent({"triggered_by": 2, "metadata": {"triggered_by": None}}) == (2, None) + assert causal_parent({"triggered_by": 2, "metadata": {"triggered_by": 1}}) == ( + None, "conflicting triggered_by (top-level 2 vs metadata 1)") + + +def test_bool_seq_ids_are_not_counted(): + # previously Python counted True as seq 1 and Rust/JS did not — a verdict split + assert verify_dag([{"seq_id": True}, {"seq_id": 1}]) == [] + assert verify_dag([{"seq_id": 1}, {"seq_id": 2, "triggered_by": 1}, {"seq_id": "x", "triggered_by": 1}]) == [ + "seq ?: triggered_by 1 cannot be ordered (seq_id is not an integer)" + ] + + +def _evt(): + return agent_tool_call_event(source_agent="agent:t", tool_name="t", args={}, result={}, success=True, duration_ms=0) + + +@pytest.mark.parametrize("bad", [True, "1", 1.0]) +def test_writer_rejects_non_integer_triggered_by(tmp_path, bad): + w = LedgerWriter(tmp_path / "l.jsonl") + w.append(event=_evt(), actor_id="korg:x") + with pytest.raises(CausalityError): + w.append(event=_evt(), actor_id="korg:x", triggered_by=bad) + + +def test_writer_output_passes_the_nested_dag_check(tmp_path): + led = tmp_path / "l.jsonl" + w = LedgerWriter(led) + a = w.append(event=_evt(), actor_id="korg:x") + b = w.append(event=_evt(), actor_id="korg:x", triggered_by=a) + w.append(event=_evt(), actor_id="korg:x", triggered_by=b) + events = [json.loads(l) for l in led.read_text().splitlines()] + assert verify_dag(events) == [] + # and a hand-edited forward link in metadata is now caught + events[1]["metadata"]["triggered_by"] = 3 + assert verify_dag(events) == ["seq 2: triggered_by 3 is not strictly earlier"] diff --git a/adapters/korg-ledger-py/tests/test_conformance.py b/adapters/korg-ledger-py/tests/test_conformance.py index f8c04f2..28b318e 100644 --- a/adapters/korg-ledger-py/tests/test_conformance.py +++ b/adapters/korg-ledger-py/tests/test_conformance.py @@ -3,7 +3,7 @@ import pytest -from korg_ledger import chain_hash, verify_chain +from korg_ledger import chain_hash, verify_chain, verify_dag # tests/ -> korg-ledger-py -> adapters -> SPEC = Path(__file__).resolve().parents[3] / "spec" / "korg-ledger-v1" @@ -16,7 +16,7 @@ def _vectors(): def _read(name): - text = (SPEC / "vectors" / name).read_text() + text = (SPEC / "vectors" / name).read_text(encoding="utf-8") return [json.loads(line) for line in text.splitlines() if line.strip()] @@ -48,3 +48,10 @@ def test_detects_tampering(): errors = verify_chain(events, key) assert errors, f"{v['file']} should report tampering" assert any(v["error_contains"] in e for e in errors), errors + + +@pytest.mark.skipif(not SPEC.exists(), reason="spec vectors not present") +def test_dag_errors_match_the_manifest_exactly(): + for v in _vectors(): + want = v.get("dag_errors", []) if v.get("dag") == "broken" else [] + assert verify_dag(_read(v["file"])) == want, v["file"] diff --git a/adapters/korg-ledger-py/tests/test_imports.py b/adapters/korg-ledger-py/tests/test_imports.py new file mode 100644 index 0000000..c3c3106 --- /dev/null +++ b/adapters/korg-ledger-py/tests/test_imports.py @@ -0,0 +1,217 @@ +"""korg-import@v1: Okto diagnostics + ningen-shikkaku lifecycle envelopes.""" +import json +from pathlib import Path + +import pytest + +from korg_ledger import LedgerWriter, verify_chain, verify_dag, verify_redaction +from korg_ledger.__main__ import main as cli +from korg_ledger.imports import ( + ImportRejected, + dazai_log_to_envelope, + envelope_digest, + import_envelope, + okto_mutation_record, + okto_session_to_envelope, + validate_envelope, +) + +IMPORTS = Path(__file__).resolve().parents[3] / "spec" / "korg-ledger-v1" / "imports" +SECRET = "sk-ant-api03-REALSECRETREALSECRET0000" + + +def _load(name): + return json.loads((IMPORTS / name).read_text(encoding="utf-8")) + + +def _events(path): + return [json.loads(l) for l in Path(path).read_text().splitlines() if l.strip()] + + +@pytest.mark.parametrize("name", ["okto-diagnostics.example.json", "ningen-shikkaku-lifecycle.example.json"]) +def test_spec_examples_import_into_a_verifiable_ledger(tmp_path, name): + env = _load(name) + assert validate_envelope(env) == [] + led = tmp_path / "l.jsonl" + w = LedgerWriter(led) + root = w.append(event={"event_type": "AgentToolCall", "tool_name": "user_prompt"}, actor_id="korg:x") + seqs = import_envelope(w, env, triggered_by=root) + events = _events(led) + assert seqs == list(range(2, 2 + 1 + len(env["records"]))) + assert verify_chain(events) == [] and verify_dag(events) == [] and verify_redaction(events) == [] + batch, first = events[1], events[2] + assert batch["event"]["tool_name"] == "korg.import" + assert batch["event"]["args"]["envelope_sha256"] == envelope_digest(env) + assert batch["metadata"]["triggered_by"] == root + src = env["source"] + assert first["event"]["source_agent"] == f"import:{src['name']}@{src['version']}" + assert first["event"]["tool_name"] == f"{src['name']}.{env['records'][0]['kind']}" + assert all(e["metadata"]["triggered_by"] == seqs[0] for e in events[2:]) + + +@pytest.mark.parametrize( + "name, expected", + [ + ("okto-diagnostics.rejected.json", [ + "records[0]: forbidden field 'request_headers' (may carry secrets or raw traffic)", + "records[1]: forbidden field 'description' (may carry secrets or raw traffic)", + ]), + ("ningen-shikkaku-lifecycle.rejected.json", [ + "records[0]: forbidden field 'value' (may carry secrets or raw traffic)", + "records[1]: field 'reason' has an invalid value", + ]), + ], +) +def test_spec_rejections_are_atomic(tmp_path, name, expected): + env = _load(name) + errors = validate_envelope(env) + for want in expected: + assert any(e.startswith(want) for e in errors), (want, errors) + led = tmp_path / "l.jsonl" + with pytest.raises(ImportRejected): + import_envelope(LedgerWriter(led), env) + assert led.read_text() == "" # nothing appended, not even the batch event + + +def test_errors_never_echo_values(): + env = _load("okto-diagnostics.example.json") + env["records"][0]["client"] = SECRET + env["records"][1]["kind"] = SECRET + env["profile"] = "okto.diagnostics@v1" + assert not any(SECRET in e for e in validate_envelope(env)) + + +@pytest.mark.parametrize( + "mutate, needle", + [ + (lambda e: e.update(schema="korg-import@v2"), "schema must be"), + (lambda e: e.update(profile="okto.raw@v1"), "unknown profile"), + (lambda e: e["source"].update(name="ningen-shikkaku"), "source.name must be 'okto'"), + (lambda e: e["source"].update(extra=1), "source must be exactly"), + (lambda e: e.update(records=[]), "non-empty"), + (lambda e: e.update(note="hi"), "unknown field 'note'"), + (lambda e: e["records"][1].update(duration_ms=1840.5), "'duration_ms' has an invalid value"), + (lambda e: e["records"][1].update(status_code=True), "'status_code' has an invalid value"), + (lambda e: e["records"][0].pop("client"), "missing required field 'client'"), + (lambda e: e["records"][0].update(session_id="has spaces/and slashes"), "'session_id' has an invalid value"), + (lambda e: e["records"][3].update(mutation_id=SECRET), "secret-shaped value at /mutation_id"), + ], +) +def test_allowlist_rejections(mutate, needle): + env = _load("okto-diagnostics.example.json") + mutate(env) + errors = validate_envelope(env) + assert any(needle in e for e in errors), errors + + +def _okto_session(): + """Shaped like okto's selfhswarm.models.Session — full of secret-bearing fields.""" + return { + "id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", + "client": "claude", + "node_id": "node-1", + "started_at_hlc": {"wall_ms": 1781395200000, "counter": 0, "node_id": "node-1"}, + "connect_target": "internal-gateway.corp:443", + "tls_established": True, + "provider": {"vendor": "Anthropic", "product": "Messages API", "hostname": "api.anthropic.com"}, + "turns": [{ + "index": 0, + "request_headers": {"authorization": f"Bearer {SECRET}", "x-api-key": SECRET}, + "request_body": {"messages": [{"role": "user", "content": f"my key is {SECRET}"}]}, + "response_headers": {"set-cookie": "session=abc"}, + "response_status": 200, + "stream_chunks": [ + {"index": 0, "raw_hex": SECRET.encode().hex(), "parsed": {"t": SECRET}, "anomalous": False}, + {"index": 1, "raw_hex": "7b", "anomalous": True, "anomaly_type": "malformed_json_in_sse_frame", + "applied_mutation_id": "m-1", "expected_hex": SECRET.encode().hex()}, + {"index": 2, "raw_hex": "", "anomalous": True, "anomaly_type": "brand_new_label"}, + ], + "token_usage": {"input_tokens": 12, "output_tokens": 3}, + "duration_ms": 1840.6, + "anomalies_detected": 2, + "mutations_applied": ["m-1"], + }], + } + + +def test_okto_session_projection_never_copies_secret_bearing_fields(tmp_path): + env = okto_session_to_envelope(_okto_session(), okto_version="0.1.0") + blob = json.dumps(env) + for leaked in (SECRET, SECRET.encode().hex(), "internal-gateway", "api.anthropic.com", "session=abc"): + assert leaked not in blob + assert validate_envelope(env) == [] + kinds = [r["kind"] for r in env["records"]] + assert kinds == ["session", "turn", "anomaly", "anomaly"] + session, turn, a1, a2 = env["records"] + assert session["provider_vendor"] == "anthropic" and session["provider_product"] == "messages-api" + assert (session["turn_count"], session["anomaly_count"], session["mutation_count"]) == (1, 2, 1) + assert turn["duration_ms"] == 1841 and turn["status_code"] == 200 and turn["chunk_count"] == 3 + assert (a1["chunk_index"], a1["repaired"], a2["anomaly_type"], a2["repaired"]) == (1, True, "other", False) + led = tmp_path / "l.jsonl" + import_envelope(LedgerWriter(led), env) + assert SECRET not in led.read_text() + + +def test_okto_mutation_projection_drops_traffic_bytes(): + rec = okto_mutation_record({ + "id": "9d8c7b6a-5e4f-4a3b-8c2d-1e0f9a8b7c6d", "hlc": {"wall_ms": 5, "counter": 0, "node_id": "n"}, + "client": "codex", "scope": "stream_chunk", "pattern_hex": SECRET.encode().hex(), + "patch_hex": "00", "patch_type": "literal", "description": f"JSON parse failure: {SECRET}", + "confidence": 0.92345, "source_node": "n", "session_id": "s-1", "reinforcement_count": 2, + }) + assert rec == {"kind": "mutation", "mutation_id": "9d8c7b6a-5e4f-4a3b-8c2d-1e0f9a8b7c6d", "scope": "stream_chunk", + "patch_type": "literal", "at_ms": 5, "session_id": "s-1", "client": "codex", + "confidence_bp": 9234, "reinforcement_count": 2} + + +DAZAI_LOG = f"""\ +[dazai] watchdog pid=4242 mode=ARMED — real wipe + SIGKILL on trigger +[dazai] socket=/run/user/1000/dazai.sock +[dazai] allocated 2 secret buffer(s), 2 mlock'd +[dazai] registered pid 4300 (1 total) +[dazai] ARMED trigger (heartbeat connection dropped): SIGKILL in 5s unless a client reconnects or sends CANCEL +[dazai] panic CANCELLED (client reconnected) +[dazai] DRY-RUN HARD trigger (SIGUSR2 hard panic): zeroizing buffers; WOULD SIGKILL registered PIDs and self +[dazai] PANIC (grace window expired): SIGKILL registered PIDs, zeroize buffers, then SIGKILL self +[dazai] SIGKILL registered pid 4300: already gone +[dazai] wiped 2 secret buffer(s) +motokano: parsing --tool "name=key,kind=static,value={SECRET}" +12:00:01 [dazai.deadman] DRY-RUN: zeroized 2 buffer(s); standing down (no kill) +[dazai] client QUIT: intentional clean stand-down +""" + + +def test_dazai_log_projection_extracts_lifecycle_and_drops_everything_else(tmp_path): + env = dazai_log_to_envelope(DAZAI_LOG.splitlines(), version="0.1.0") + assert SECRET not in json.dumps(env) and "/run/user" not in json.dumps(env) + assert env["dropped"] == 2 # the socket line and the motokano error that echoes a tool spec + assert env["records"] == [ + {"kind": "daemon_start", "pid": 4242, "mode": "armed"}, + {"kind": "buffers", "allocated": 2, "locked": 2}, + {"kind": "pid_registered", "pid": 4300, "total": 1}, + {"kind": "trigger", "mode": "armed", "severity": "graceful", "reason": "heartbeat_dropped"}, + {"kind": "cancelled", "reason": "client_reconnected"}, + {"kind": "trigger", "mode": "dry_run", "severity": "hard", "reason": "sigusr2"}, + {"kind": "panic", "reason": "grace_expired"}, + {"kind": "pid_killed", "pid": 4300, "outcome": "already_gone"}, + {"kind": "wiped", "buffers": 2}, + {"kind": "wiped", "buffers": 2}, + {"kind": "shutdown", "how": "quit"}, + ] + assert validate_envelope(env) == [] + + +def test_cli_round_trip(tmp_path, capsys): + log = tmp_path / "daemon.log" + log.write_text(DAZAI_LOG) + assert cli(["envelope", "dazai-log", str(log), "--source-version", "0.1.0"]) == 0 + env_path = tmp_path / "env.json" + env_path.write_text(capsys.readouterr().out) + led = tmp_path / "ledger.jsonl" + assert cli(["import", str(env_path), "--ledger", str(led)]) == 0 + assert cli(["lint", str(led)]) == 0 + bad = tmp_path / "bad.json" + bad.write_text((IMPORTS / "ningen-shikkaku-lifecycle.rejected.json").read_text()) + before = led.read_text() + assert cli(["import", str(bad), "--ledger", str(led)]) == 1 + assert led.read_text() == before diff --git a/adapters/korg-ledger-py/tests/test_redaction.py b/adapters/korg-ledger-py/tests/test_redaction.py new file mode 100644 index 0000000..905cb3c --- /dev/null +++ b/adapters/korg-ledger-py/tests/test_redaction.py @@ -0,0 +1,135 @@ +"""korg-redact@v1: the shared hand-authored cases + redaction-at-persistence.""" +import json +from pathlib import Path + +import pytest +from hypothesis import given, settings, strategies as st + +from korg_ledger import ( + LedgerWriter, + agent_tool_call_event, + redact, + scan, + verify_chain, + verify_dag, + verify_redaction, +) +from korg_ledger.redaction import finding_order + +SPEC = Path(__file__).resolve().parents[3] / "spec" / "korg-ledger-v1" +CASES = json.loads((SPEC / "vectors" / "redaction" / "korg-redact-v1.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("case", CASES["cases"], ids=lambda c: c["name"]) +def test_shared_redaction_cases(case): + want = case["input"] if case["output"] is None else case["output"] + got, findings = redact(case["input"]) + assert got == want + assert sorted(findings, key=finding_order) == case["findings"] + # idempotent: the output is clean and a second pass changes nothing + assert redact(got) == (got, []) + + +# Fragments that can splice into secrets or near-secrets; idempotence must hold +# for any concatenation, including ones that create overlaps between rules. +_frag = st.sampled_from([ + "sk-", "abcdefghijklmnopqrstuvwxyz", "Bearer ", "API_KEY", "=", ":", " ", "\n", "'", '"', + "https://", "u:p@", "@", "AKIA", "IOSFODNN7EXAMPLE", "-----BEGIN RSA PRIVATE KEY-----", + "-----END RSA PRIVATE KEY-----", "eyJ", "abcdefgh.", '"password": ', "[REDACTED:jwt]", "é", "ghp_", + "0123456789" * 4, "xoxb-", "AIza", +]) + + +@settings(max_examples=3000, deadline=None) +@given(st.lists(_frag, max_size=12).map("".join)) +def test_redaction_is_idempotent_on_spliced_strings(s): + once, _ = redact(s) + assert redact(once) == (once, []) + + +def _evt(args, result=None): + return agent_tool_call_event( + source_agent="agent:claude-code@2.1.0", tool_name="Bash", args=args, + result=result or {}, success=True, duration_ms=1, + ) + + +SECRET = "sk-proj-THISISASECRETVALUE0123456789" + + +def test_writer_redacts_before_hashing_and_never_writes_the_secret(tmp_path): + led = tmp_path / "ledger.jsonl" + w = LedgerWriter(led) + s1 = w.append(event=_evt({"command": f"export OPENAI_API_KEY={SECRET}"}), actor_id="korg:claude-hook") + w.append( + event=_evt({"url": "https://bob:hunter2@example.com"}, {"headers": {"Authorization": f"Bearer {SECRET}"}}), + actor_id="korg:claude-hook", triggered_by=s1, + ) + raw = led.read_text() + assert SECRET not in raw and "hunter2" not in raw + events = [json.loads(l) for l in raw.splitlines()] + # the chain is over the redacted bytes, so it verifies as-is + assert verify_chain(events) == [] and verify_dag(events) == [] + assert verify_redaction(events) == [] + assert events[0]["event"]["args"]["command"] == "export OPENAI_API_KEY=[REDACTED:env_assignment]" + assert events[1]["event"]["result"]["headers"]["Authorization"] == "[REDACTED:sensitive_key]" + assert all(e["metadata"]["tags"]["korg.redaction"] == "korg-redact@v1" for e in events) + + +def test_writer_does_not_mutate_the_callers_event(tmp_path): + ev = _evt({"api_key": "k"}) + LedgerWriter(tmp_path / "l.jsonl").append(event=ev, actor_id="korg:x") + assert ev["args"]["api_key"] == "k" + + +def test_opting_out_is_visible_to_the_lint(tmp_path): + led = tmp_path / "l.jsonl" + LedgerWriter(led, redaction=None).append(event=_evt({"command": f"X={SECRET}"}), actor_id="korg:x") + events = [json.loads(l) for l in led.read_text().splitlines()] + assert "korg.redaction" not in events[0]["metadata"]["tags"] + assert verify_redaction(events) == ["seq 1: unredacted provider_api_key at /event/args/command"] + + +def test_unknown_policy_is_rejected(tmp_path): + with pytest.raises(ValueError): + LedgerWriter(tmp_path / "l.jsonl", redaction="korg-redact@v9") + + +def test_lint_matches_the_conformance_manifest(): + manifest = json.loads((SPEC / "conformance.json").read_text(encoding="utf-8")) + checked = 0 + for v in manifest["vectors"]: + if "redaction" not in v: + continue + lines = (SPEC / "vectors" / v["file"]).read_text(encoding="utf-8").splitlines() + events = [json.loads(l) for l in lines if l.strip()] + want = v.get("redaction_errors", []) if v["redaction"] == "unclean" else [] + assert verify_redaction(events) == want, v["file"] + checked += 1 + assert checked >= 2 + + +def test_rechain_redacted_migrates_an_unredacted_journal(tmp_path): + from korg_ledger.__main__ import main as cli + from korg_ledger.redaction import rechain_redacted + + old = tmp_path / "old.jsonl" + w = LedgerWriter(old, redaction=None) + a = w.append(event=_evt({"command": f"X_API_KEY={SECRET}"}), actor_id="korg:x") + w.append(event=_evt({"ok": 1}), actor_id="korg:x", triggered_by=a) + events = [json.loads(l) for l in old.read_text().splitlines()] + assert verify_redaction(events) # the old journal leaks + + new = rechain_redacted(events) + assert SECRET not in json.dumps(new) + assert verify_chain(new) == [] and verify_dag(new) == [] and verify_redaction(new) == [] + assert [e["seq_id"] for e in new] == [1, 2] and new[1]["metadata"]["triggered_by"] == 1 + assert new[-1]["entry_hash"] != events[-1]["entry_hash"] # a new chain, by design + + flat = rechain_redacted([{"seq_id": 1, "args": {"token": "t"}}]) + assert flat[0]["redaction"] == "korg-redact@v1" and flat[0]["args"]["token"] == "[REDACTED:sensitive_key]" + + out = tmp_path / "new.jsonl" + assert cli(["rechain-redacted", str(old), str(out)]) == 0 + assert cli(["lint", str(out)]) == 0 + assert cli(["lint", str(old)]) == 1 diff --git a/agent_event_spec.md b/agent_event_spec.md index f0a5391..472fbcc 100644 --- a/agent_event_spec.md +++ b/agent_event_spec.md @@ -32,12 +32,30 @@ contract. ## Canonicality rule -When this spec disagrees with what Korg emits on the wire, **the wire -is canonical** and the spec gets a PR. This applies to string-level -details (e.g., `schema_version` value), field placement (append shape -vs journal shape per §1.2), and any structural shape choice. If you -find a disagreement: file the spec PR, point reviewers at the Rust -source location, and don't change Korg's behavior to match this doc. +**The ledger contract is [`spec/korg-ledger-v1/`](spec/korg-ledger-v1/SPEC.md), +and it is authoritative.** This document is the *application profile* for the +HTTP/MCP write path (event semantics, actor naming, rewind, MCP resources). Where +the two overlap — chain fields, journal shapes, where `triggered_by` lives and how +it is validated, redaction, imported evidence — `SPEC.md` (§9–§11), its vectors, +`REDACTION.md`, and `IMPORT.md` win, and a disagreement here or in code is a bug +to fix *toward the spec*. + +Within this profile's own scope, when it disagrees with what Korg emits on the +wire, file a spec PR that points reviewers at the Rust source location. + +Known corrections against the canonical spec (revision 1.1): + +- The chain fields are top-level `prev_hash` / `entry_hash` (+ optional + `event_sig`), not `metadata.ledger_hash` / `metadata.parent_hash` as §1.2.3 + below describes. +- The append endpoint is `POST /api/agent/tool-call`. Since revision 1.1 it + redacts `args`/`result` under `korg-redact@v1` before persisting, and answers + **422** when `triggered_by` does not name an existing, strictly-earlier + `seq_id` (SPEC §9.5). +- Verifiers read the causal parent from top-level `triggered_by` (flat) or + `metadata.triggered_by` (nested) — SPEC §9.3. +- A fifth actor prefix, `import:@`, marks evidence imported + through a `korg-import@v1` envelope (IMPORT.md §3). --- @@ -83,7 +101,7 @@ There are two distinct representations of an AgentToolCall event. #### §1.2.1 — Append shape (POST body) -The flat body sent by a client to `POST /api/agent_tool_call`: +The flat body sent by a client to `POST /api/agent/tool-call`: | Field | Type | Required | Notes | |---|---|---|---| @@ -213,6 +231,7 @@ ignores them if present. | `human:` | Human override | `human:dusk` | | `korg:` | Korg internal event | `korg:replay` | | `mcp:` | MCP server client | `mcp:github` | +| `import:@` | Evidence imported via a `korg-import@v1` envelope (canonical spec IMPORT.md) — attested by the importer, not observed by korg | `import:okto@0.1.0` | Sub-agents use distinct identities; see §2b for the convention under the cross-agent spine. diff --git a/crates/korg-bridge/Cargo.toml b/crates/korg-bridge/Cargo.toml index 61eade1..2221475 100644 --- a/crates/korg-bridge/Cargo.toml +++ b/crates/korg-bridge/Cargo.toml @@ -22,6 +22,7 @@ extension-module = ["pyo3/extension-module"] [dependencies] korg-core = { path = "../korg-core" } korg-registry = { path = "../korg-registry" } +korg-ledger = { path = "../korg-ledger", features = ["redact"] } serde = { workspace = true } serde_json = { workspace = true } chrono = { workspace = true } diff --git a/crates/korg-bridge/src/lib.rs b/crates/korg-bridge/src/lib.rs index fe1d852..7b6a0a4 100644 --- a/crates/korg-bridge/src/lib.rs +++ b/crates/korg-bridge/src/lib.rs @@ -414,6 +414,10 @@ impl Bridge { } } + // Redaction at persistence (spec/korg-ledger-v1/REDACTION.md) — the + // same gate as the HTTP handler: redact before the journal sees it. + let (args, _) = korg_ledger::redact::redact(&args); + let (result, _) = korg_ledger::redact::redact(&result); let event = CapabilityEvent::AgentToolCall { source_agent: source_agent.to_string(), tool_name: tool_name.to_string(), @@ -460,7 +464,10 @@ impl Bridge { retry_count: 0, tier: EventTier::Telemetry, span_id: None, - tags: std::collections::BTreeMap::new(), + tags: std::collections::BTreeMap::from([( + korg_ledger::redact::TAG.to_string(), + korg_ledger::redact::POLICY.to_string(), + )]), triggered_by, }; diff --git a/crates/korg-ledger/Cargo.toml b/crates/korg-ledger/Cargo.toml index 54fa2c8..0af2af0 100644 --- a/crates/korg-ledger/Cargo.toml +++ b/crates/korg-ledger/Cargo.toml @@ -16,11 +16,15 @@ sha2 = "0.10" hmac = "0.12" ed25519-dalek = { version = "2", optional = true } hex = { version = "0.4", optional = true } +regex = { version = "1", optional = true } [features] # Opt-in Ed25519 per-event signing/verification. Keeps the core crate # dependency-light (hash-chain only) for downstream consumers who don't sign. signing = ["dep:ed25519-dalek", "dep:hex"] +# Opt-in korg-redact@v1 (redaction at persistence + the cleanliness lint). +# Producers that persist events and verifiers that lint them enable it. +redact = ["dep:regex"] [dev-dependencies] proptest = "1" diff --git a/crates/korg-ledger/src/lib.rs b/crates/korg-ledger/src/lib.rs index 5038cf3..ffabb24 100644 --- a/crates/korg-ledger/src/lib.rs +++ b/crates/korg-ledger/src/lib.rs @@ -12,6 +12,9 @@ //! the chain and is localized to a `seq_id`. With an HMAC key the chain is //! tamper-PROOF (unforgeable without the key), not merely tamper-evident. +#[cfg(feature = "redact")] +pub mod redact; + use hmac::{Hmac, Mac}; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -255,32 +258,107 @@ pub fn verify_chain(events: &[Value], key: Option<&[u8]>) -> Vec { errors } -/// Check the causal DAG is well-formed (korg-ledger@v1 §5): unique `seq_id`s, -/// and every `triggered_by` references an existing, strictly-earlier `seq_id`. +/// The integer a JSON value denotes, or `None` (korg-ledger@v1 §9.2). +/// +/// A ledger integer is a JSON number with an integral value within ±(2^53-1). +/// Booleans are not integers, and an integral float such as `2.0` IS the integer +/// 2 — JS cannot tell `2.0` from `2` after parsing, so all three verifiers accept +/// it in order to agree. +pub fn json_int(value: &Value) -> Option { + let n = value.as_number()?; + if let Some(i) = n.as_i64() { + return ((i as i128).abs() <= SAFE_INT_MAX).then_some(i); + } + if n.as_u64().is_some() { + return None; // > i64::MAX, far outside the safe range + } + let f = n.as_f64()?; + (f.fract() == 0.0 && f.abs() <= SAFE_INT_MAX as f64).then_some(f as i64) +} + +fn seq_label(e: &Value) -> String { + e.get("seq_id") + .and_then(json_int) + .map(|n| n.to_string()) + .unwrap_or_else(|| "?".to_string()) +} + +/// Resolve an event's causal parent across the flat and nested journal shapes +/// (korg-ledger@v1 §9.3). A flat record carries `triggered_by` at the top level; +/// a nested JournalEvent carries it under `metadata`. `null` and absence both +/// mean "root". When both are present they must name the same parent. +/// +/// `Ok(None)` = root, `Ok(Some(p))` = parent seq_id, `Err(msg)` = malformed link +/// (message without the `seq N:` prefix). +pub fn causal_parent(event: &Value) -> Result, String> { + let mut raw: Vec<&Value> = Vec::new(); + if let Some(v) = event.get("triggered_by").filter(|v| !v.is_null()) { + raw.push(v); + } + if let Some(v) = event + .get("metadata") + .filter(|m| m.is_object()) + .and_then(|m| m.get("triggered_by")) + .filter(|v| !v.is_null()) + { + raw.push(v); + } + let mut parsed = Vec::with_capacity(2); + for v in raw { + match json_int(v) { + Some(n) => parsed.push(n), + None => return Err("triggered_by is not an integer".to_string()), + } + } + match parsed.as_slice() { + [] => Ok(None), + [a, b] if a != b => Err(format!( + "conflicting triggered_by (top-level {a} vs metadata {b})" + )), + [a, ..] => Ok(Some(*a)), + } +} + +/// Check the causal DAG is well-formed (korg-ledger@v1 §5 + §9): unique integer +/// `seq_id`s, and every causal parent — top-level `triggered_by` (flat) or +/// `metadata.triggered_by` (nested JournalEvent) — is a well-formed integer that +/// references an existing, strictly-earlier `seq_id`. Error strings are +/// byte-identical to the Python and JS verifiers (pinned by the conformance +/// manifest). pub fn verify_dag(events: &[Value]) -> Vec { let mut errors = Vec::new(); - let seqs: Vec = events + let objs: Vec<&Value> = events.iter().filter(|e| e.is_object()).collect(); + let seqs: Vec = objs .iter() - .filter_map(|e| e.get("seq_id").and_then(|v| v.as_i64())) + .filter_map(|e| e.get("seq_id").and_then(json_int)) .collect(); let seqset: std::collections::HashSet = seqs.iter().copied().collect(); if seqset.len() != seqs.len() { errors.push("duplicate seq_id present".to_string()); } - for e in events { - let tb = match e.get("triggered_by").and_then(|v| v.as_i64()) { - Some(tb) => tb, - None => continue, + for e in objs { + let label = seq_label(e); + let tb = match causal_parent(e) { + Ok(Some(tb)) => tb, + Ok(None) => continue, + Err(msg) => { + errors.push(format!("seq {label}: {msg}")); + continue; + } }; - let sid = e.get("seq_id").and_then(|v| v.as_i64()); + let sid = e.get("seq_id").and_then(json_int); if !seqset.contains(&tb) { - errors.push(format!("seq {sid:?}: triggered_by {tb} does not exist")); + errors.push(format!("seq {label}: triggered_by {tb} does not exist")); } else if let Some(sid) = sid { if tb >= sid { errors.push(format!( - "seq {sid}: triggered_by {tb} is not strictly earlier" + "seq {label}: triggered_by {tb} is not strictly earlier" )); } + } else { + errors.push(format!( + "seq ?: triggered_by {tb} cannot be ordered (seq_id is not an integer)" + )); } } errors diff --git a/crates/korg-ledger/src/redact.rs b/crates/korg-ledger/src/redact.rs new file mode 100644 index 0000000..a05d4bc --- /dev/null +++ b/crates/korg-ledger/src/redact.rs @@ -0,0 +1,378 @@ +//! korg-redact@v1 — redaction at persistence (Rust reference implementation). +//! +//! Normative text: `spec/korg-ledger-v1/REDACTION.md`. Byte-for-byte equivalent +//! to the Python `korg_ledger.redaction` module; both are pinned by the shared +//! cases in `spec/korg-ledger-v1/vectors/redaction/korg-redact-v1.json`. +//! +//! A chained event cannot be redacted after it is hashed (that is tampering), so +//! a producer MUST run [`redact`] on the record *before* [`crate::chain_hash`] and +//! before anything touches disk. The policy is deterministic and idempotent +//! (`redact(redact(x)) == redact(x)`), and a value is redaction-clean iff +//! [`scan`] returns no findings. +//! +//! The patterns use ASCII classes only (no `\s`, `\b`, `\w`, or `(?i)`): the +//! `regex` crate and Python `re` disagree on their Unicode semantics, and a split +//! would let one implementation persist what the other would redact. + +use regex::{Captures, Regex}; +use serde_json::{Map, Value}; +use std::sync::OnceLock; + +/// Policy identifier recorded on every redacted event. +pub const POLICY: &str = "korg-redact@v1"; + +/// Tag key under `metadata.tags` that declares the policy a producer applied. +pub const TAG: &str = "korg.redaction"; + +/// Fields that are chain structure, never payload: excluded from redaction and +/// from the cleanliness lint. +pub const STRUCTURAL_FIELDS: &[&str] = &["prev_hash", "entry_hash", "event_sig"]; + +const SENSITIVE_KEYS: &[&str] = &[ + "authorization", + "proxy_authorization", + "cookie", + "set_cookie", + "api_key", + "apikey", + "password", + "passwd", + "passphrase", + "secret", + "token", + "private_key", + "secret_key", + "aws_secret_access_key", + "credential", + "credentials", +]; + +const SENSITIVE_KEY_SUFFIXES: &[&str] = &[ + "_api_key", + "_apikey", + "_secret", + "_secret_key", + "_private_key", + "_password", + "_passwd", + "_passphrase", + "_token", +]; + +/// One redaction: an RFC 6901 JSON pointer and the rule that fired there. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Finding { + pub path: String, + pub rule: &'static str, +} + +/// The replacement text for a rule. +pub fn marker(rule: &str) -> String { + format!("[REDACTED:{rule}]") +} + +/// camelCase / kebab-case / spaced → snake_case, ASCII-lowercased +/// (`accessToken` → `access_token`, `X-Api-Key` → `x_api_key`, `APIKey` → `apikey`). +pub fn normalize_key(key: &str) -> String { + let mut out = String::with_capacity(key.len() + 4); + let mut prev: Option = None; + for c in key.chars() { + if c.is_ascii_uppercase() { + if prev.is_some_and(|p| p.is_ascii_lowercase() || p.is_ascii_digit()) { + out.push('_'); + } + out.push(c.to_ascii_lowercase()); + } else if c == '-' || c == ' ' { + out.push('_'); + } else { + out.push(c); + } + prev = Some(c); + } + out +} + +/// Whether an object member with this key holds a credential. +pub fn is_sensitive_key(key: &str) -> bool { + let k = normalize_key(key); + SENSITIVE_KEYS.contains(&k.as_str()) || SENSITIVE_KEY_SUFFIXES.iter().any(|s| k.ends_with(s)) +} + +type Replacer = fn(&Captures) -> String; + +struct Rule { + id: &'static str, + re: Regex, + repl: Replacer, +} + +// Leading "boundary": start of string or a non-[A-Za-z0-9_] char, captured and +// re-emitted (the regex crate has no look-behind). +const B: &str = r"(^|[^A-Za-z0-9_])"; + +fn rules() -> &'static [Rule] { + static RULES: OnceLock> = OnceLock::new(); + RULES.get_or_init(|| { + let r = |id: &'static str, pat: String, repl: Replacer| Rule { + id, + re: Regex::new(&pat).expect("korg-redact@v1 pattern compiles"), + repl, + }; + vec![ + r( + "private_key_block", + r"-----BEGIN[A-Z ]*PRIVATE KEY-----(?:[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----|[\s\S]*)" + .to_string(), + |_| marker("private_key_block"), + ), + r( + "url_credentials", + r"([A-Za-z][A-Za-z0-9+.\-]*://)[^ \t\r\n/:@\[\]]+:[^ \t\r\n/@\[\]]+@".to_string(), + |c| format!("{}{}@", &c[1], marker("url_credentials")), + ), + r( + "bearer_token", + r"((?:Bearer|bearer|BEARER)[ \t]+)[A-Za-z0-9._~+/\-]{16,}=*".to_string(), + |c| format!("{}{}", &c[1], marker("bearer_token")), + ), + r( + "provider_api_key", + format!(r"{B}sk-[A-Za-z0-9_\-]{{20,}}"), + |c| format!("{}{}", &c[1], marker("provider_api_key")), + ), + r( + "github_token", + format!(r"{B}(?:gh[pousr]_[A-Za-z0-9]{{36,}}|github_pat_[A-Za-z0-9_]{{22,}})"), + |c| format!("{}{}", &c[1], marker("github_token")), + ), + r( + "slack_token", + format!(r"{B}xox[abposr]-[A-Za-z0-9\-]{{10,}}"), + |c| format!("{}{}", &c[1], marker("slack_token")), + ), + r( + "aws_access_key_id", + format!(r"{B}(?:AKIA|ASIA)[0-9A-Z]{{16}}"), + |c| format!("{}{}", &c[1], marker("aws_access_key_id")), + ), + r( + "google_api_key", + format!(r"{B}AIza[0-9A-Za-z_\-]{{35}}"), + |c| format!("{}{}", &c[1], marker("google_api_key")), + ), + r( + "jwt", + format!(r"{B}eyJ[A-Za-z0-9_\-]{{8,}}\.eyJ[A-Za-z0-9_\-]{{8,}}\.[A-Za-z0-9_\-]{{8,}}"), + |c| format!("{}{}", &c[1], marker("jwt")), + ), + r( + "env_assignment", + format!( + r#"{B}([A-Z][A-Z0-9_]*(?:API_KEY|APIKEY|SECRET|SECRET_KEY|PASSWORD|PASSWD|TOKEN|PRIVATE_KEY|ACCESS_KEY))([ \t]*[=:][ \t]*)(?:"[^"\n]*"|'[^'\n]*'|[^ \t\r\n"',;&]+)"# + ), + |c| format!("{}{}{}{}", &c[1], &c[2], &c[3], marker("env_assignment")), + ), + r( + "json_secret_field", + r#"("(?:password|passwd|passphrase|secret|client_secret|api_key|apikey|access_token|refresh_token|auth_token|id_token|token|private_key|authorization)"[ \t]*:[ \t]*)"(?:[^"\\\n]|\\.)*""# + .to_string(), + |c| format!("{}\"{}\"", &c[1], marker("json_secret_field")), + ), + ] + }) +} + +fn marker_re() -> &'static Regex { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Regex::new(r"^\[REDACTED:[a-z_]+\]$").expect("marker pattern")) +} + +/// Upper bound on rule-list passes. Real input reaches its fixed point in one or +/// two passes; hitting the bound leaves the string unclean, which a producer's +/// fail-closed gate (REDACTION.md R4) then refuses to persist. +pub const MAX_PASSES: usize = 8; + +/// Apply the value-pattern rules, in order, repeatedly until the string stops +/// changing (a fixed point — idempotent by construction even where one +/// replacement exposes a new match, e.g. two keys back to back). Returns the +/// redacted string and the ids of the rules that fired, in normative order. +pub fn redact_string(s: &str) -> (String, Vec<&'static str>) { + let mut cur = s.to_string(); + let mut fired: Vec<&'static str> = Vec::new(); + for _ in 0..MAX_PASSES { + let mut changed = false; + for rule in rules() { + let new = rule.re.replace_all(&cur, |c: &Captures| (rule.repl)(c)); + if new != cur { + if !fired.contains(&rule.id) { + fired.push(rule.id); + } + cur = new.into_owned(); + changed = true; + } + } + if !changed { + break; + } + } + let order = rule_ids(); + fired.sort_by_key(|r| order.iter().position(|x| x == r)); + (cur, fired) +} + +fn pointer_escape(token: &str) -> String { + token.replace('~', "~0").replace('/', "~1") +} + +fn keeps_under_sensitive_key(v: &Value) -> bool { + match v { + Value::Null | Value::Bool(_) => true, + Value::String(s) => s.is_empty() || marker_re().is_match(s), + _ => false, + } +} + +fn redact_inner(v: &Value, path: &str, findings: &mut Vec) -> Value { + match v { + Value::String(s) => { + let (new, fired) = redact_string(s); + for rule in fired { + findings.push(Finding { + path: path.to_string(), + rule, + }); + } + Value::String(new) + } + Value::Array(a) => Value::Array( + a.iter() + .enumerate() + .map(|(i, x)| redact_inner(x, &format!("{path}/{i}"), findings)) + .collect(), + ), + Value::Object(m) => { + let mut out = Map::with_capacity(m.len()); + for (k, x) in m { + let p = format!("{path}/{}", pointer_escape(k)); + if is_sensitive_key(k) && !keeps_under_sensitive_key(x) { + findings.push(Finding { + path: p, + rule: "sensitive_key", + }); + out.insert(k.clone(), Value::String(marker("sensitive_key"))); + } else { + out.insert(k.clone(), redact_inner(x, &p, findings)); + } + } + Value::Object(out) + } + other => other.clone(), + } +} + +/// Redact a JSON value under korg-redact@v1. Returns the redacted copy and one +/// finding per rule application. Object keys are never rewritten. +pub fn redact(value: &Value) -> (Value, Vec) { + let mut findings = Vec::new(); + let out = redact_inner(value, "", &mut findings); + (out, findings) +} + +/// Findings that [`redact`] would produce — empty iff `value` is clean. +pub fn scan(value: &Value) -> Vec { + redact(value).1 +} + +/// Normative rule order: `sensitive_key`, then the value-pattern rules. +pub fn rule_ids() -> Vec<&'static str> { + std::iter::once("sensitive_key") + .chain(rules().iter().map(|r| r.id)) + .collect() +} + +/// Canonical finding order: JSON pointer (code point), then normative rule order. +/// Implementations iterate object members differently, so reported lists are +/// always sorted by this. +pub fn finding_order(a: &Finding, b: &Finding) -> std::cmp::Ordering { + let ids = rule_ids(); + let idx = |r: &str| ids.iter().position(|x| *x == r).unwrap_or(usize::MAX); + a.path + .cmp(&b.path) + .then_with(|| idx(a.rule).cmp(&idx(b.rule))) +} + +/// Lint one persisted ledger record (flat or nested) for unredacted secrets, +/// ignoring the structural hash fields. +pub fn scan_record(record: &Value) -> Vec { + match record.as_object() { + Some(m) => { + let body: Map = m + .iter() + .filter(|(k, _)| !STRUCTURAL_FIELDS.contains(&k.as_str())) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + scan(&Value::Object(body)) + } + None => Vec::new(), + } +} + +/// Journal-level lint: one error per finding, localized to `seq N` and a JSON +/// pointer. Empty iff every event is redaction-clean. Strings are byte-identical +/// to Python `verify_redaction`. +pub fn verify_redaction(events: &[Value]) -> Vec { + let mut errors = Vec::new(); + for e in events.iter().filter(|e| e.is_object()) { + let label = e + .get("seq_id") + .and_then(crate::json_int) + .map(|n| n.to_string()) + .unwrap_or_else(|| "?".to_string()); + let mut findings = scan_record(e); + findings.sort_by(finding_order); + for f in findings { + let path = if f.path.is_empty() { "/" } else { &f.path }; + errors.push(format!("seq {label}: unredacted {} at {path}", f.rule)); + } + } + errors +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn key_normalization_matches_the_python_reference() { + assert_eq!(normalize_key("accessToken"), "access_token"); + assert_eq!(normalize_key("X-Api-Key"), "x_api_key"); + assert_eq!(normalize_key("APIKey"), "apikey"); + assert_eq!(normalize_key("OAuth2Token"), "oauth2_token"); + assert!(is_sensitive_key("OPENAI_API_KEY")); + assert!(!is_sensitive_key("prompt_tokens")); + assert!(!is_sensitive_key("token_count")); + } + + #[test] + fn redaction_is_idempotent_and_keeps_non_secrets() { + let v = json!({ + "headers": {"Authorization": "Bearer abcdefghijklmnopqrstuvwxyz0123", "accept": "json"}, + "cmd": "export OPENAI_API_KEY=sk-proj-abcdefghijklmnopqrstuvwxyz && run", + "max_tokens": 5, + "secret": false + }); + let (r, f) = redact(&v); + assert_eq!(r["headers"]["Authorization"], "[REDACTED:sensitive_key]"); + assert_eq!(r["headers"]["accept"], "json"); + assert_eq!( + r["cmd"], + "export OPENAI_API_KEY=[REDACTED:env_assignment] && run" + ); + assert_eq!(r["max_tokens"], 5); + assert_eq!(r["secret"], false); + assert_eq!(f.len(), 3); + assert_eq!(redact(&r).0, r); + assert!(scan(&r).is_empty()); + } +} diff --git a/crates/korg-registry/README.md b/crates/korg-registry/README.md index 1e52ee1..41dbccd 100644 --- a/crates/korg-registry/README.md +++ b/crates/korg-registry/README.md @@ -77,7 +77,7 @@ toggle). State changes go through one of: This module is the **Rust reference** for the frozen `korg-ledger@v1` spec (canonical text in [`spec/korg-ledger-v1`](../../spec/korg-ledger-v1); vendored -golden vectors under `tests/conformance/`). Each event carries `prev_hash` (the +golden vectors in the canonical `spec/korg-ledger-v1/vectors/`). Each event carries `prev_hash` (the previous event's `entry_hash`, `GENESIS_HASH` for the first) and `entry_hash` (SHA-256, or HMAC-SHA256 when `KORG_LEDGER_HMAC_KEY` is set, over the canonical preimage). `canonicalize` reproduces Python's diff --git a/crates/korg-registry/tests/conformance.rs b/crates/korg-registry/tests/conformance.rs index 7ece770..b62266a 100644 --- a/crates/korg-registry/tests/conformance.rs +++ b/crates/korg-registry/tests/conformance.rs @@ -1,7 +1,9 @@ //! korg-ledger@v1 cross-language conformance. //! -//! The Python reference (korgex) froze the chain definition and a set of golden -//! vectors with FROZEN tip hashes (tests/conformance/conformance.json). This +//! The Python reference froze the chain definition and a set of golden vectors +//! with FROZEN tip hashes. They live in exactly one place — the canonical spec +//! directory `spec/korg-ledger-v1/` (conformance.json + vectors/) — and every +//! implementation reads them from there, so no copy can drift. This //! test proves the Rust implementation reproduces those exact hashes — i.e. that //! `korg-ledger@v1` is genuinely a multi-language standard, not a Python detail. //! If Rust's canonicalization or hashing diverged by a single byte, the frozen @@ -13,11 +15,14 @@ use std::fs; use std::path::PathBuf; fn cdir() -> PathBuf { - PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/conformance")) + PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../spec/korg-ledger-v1" + )) } fn read_jsonl(name: &str) -> Vec { - fs::read_to_string(cdir().join(name)) + fs::read_to_string(cdir().join("vectors").join(name)) .unwrap() .lines() .filter(|l| !l.trim().is_empty()) diff --git a/crates/korg-registry/tests/conformance/SPEC.md b/crates/korg-registry/tests/conformance/SPEC.md deleted file mode 100644 index fb0d510..0000000 --- a/crates/korg-registry/tests/conformance/SPEC.md +++ /dev/null @@ -1,115 +0,0 @@ -# korg-ledger@v1 — Tamper-Evident Cognition Ledger - -**Status:** FROZEN · **Version:** `korg-ledger@v1` · **Reference:** [`src/ledger_spec.py`](../../src/ledger_spec.py) · **Conformance:** [`conformance.json`](./conformance.json) - -This document is the normative definition of korg's tamper-evident event ledger. -It is intentionally small and language-agnostic. Any implementation — the Rust -core (`korg-registry`), the Python reference (korgex), a JS verifier — is -**conformant** iff it reproduces the frozen [conformance vectors](./vectors/) -byte-for-byte. The reference module and this document MUST agree; the vectors -are the tie-breaker. - -> The guarantee in one line: a journal is a hash-chain of events; any edit, -> deletion, insertion, or reorder is detectable and localized to a `seq_id`. -> With an HMAC key it is tamper-**proof** (unforgeable without the key), not -> merely tamper-**evident**. - -## 1. Event - -An event is a JSON object. The chain is defined over two reserved fields; all -other fields are opaque application payload (korgex uses `seq_id`, `tool_name`, -`args`, `result`, `success`, `duration_ms`, `triggered_by`, `source_agent`, -`schema_version`, …). - -| Field | Type | Meaning | -|---|---|---| -| `prev_hash` | hex string (64) | the previous event's `entry_hash`, or the genesis anchor for the first event | -| `entry_hash` | hex string (64) | this event's hash (see §3) | - -`GENESIS_HASH` = `"0" * 64` (64 zero characters). - -## 2. Canonicalization - -To hash an event, first canonicalize a JSON value to bytes. The rules (these are -what make the hash reproducible across languages): - -1. Serialize as JSON. -2. **Object keys sorted** ascending by Unicode code point. -3. **No insignificant whitespace** — item separator `,`, key/value separator `:`. -4. **Non-ASCII escaped** as `\uXXXX`; output is pure ASCII (so there is no UTF-8 - encoding ambiguity). Encode the resulting ASCII string to bytes. - -Reference: `json.dumps(value, sort_keys=True, separators=(",", ":")).encode("ascii")`. -Equivalent to RFC 8785 (JCS) for the JSON subset korg emits (objects, arrays, -strings, integers, booleans, null — no floats). - -``` -canonicalize({"z": [3, 2], "a": {"y": 1, "x": 2}}) == b'{"a":{"x":2,"y":1},"z":[3,2]}' -``` - -Non-ASCII is escaped to its `\uXXXX` form so the output is pure ASCII — e.g. a -value of `"é"` serializes to the six-character escape `"é"`, never raw UTF-8. -(See the conformance test `test_canonicalize_is_sorted_compact_ascii` for the -exact byte-level assertion.) - -## 3. `entry_hash` - -The **preimage** is the canonicalization of the event with its `entry_hash` -field removed (`prev_hash` IS included — that is what links the chain): - -``` -preimage = canonicalize({ k: v for k, v in event if k != "entry_hash" }) -entry_hash = sha256(preimage).hexdigest() # tamper-EVIDENT -entry_hash = hmac_sha256(key, preimage).hexdigest() # tamper-PROOF (key present) -``` - -Hex is lowercase. - -## 4. Chaining - -For events in journal order `e₁, e₂, … eₙ`: - -- `e₁.prev_hash == GENESIS_HASH` -- `eᵢ.prev_hash == eᵢ₋₁.entry_hash` for `i > 1` -- each `eᵢ.entry_hash == chain_hash(eᵢ)` per §3 - -## 5. Verification - -`verify_chain(events, key=None) -> errors[]` ( `[]` ⇔ intact ). Walk events in -order, tracking `expected_prev` (starts at `GENESIS_HASH`): - -- if `entry_hash` is absent → error "not chained"; -- if `prev_hash != expected_prev` → error "broken link" (insert/delete/reorder); -- if `chain_hash(event, key) != entry_hash` → error "content tampered"; -- set `expected_prev = entry_hash`. - -Each error names the offending `seq_id`. A verifier given the wrong key (or no -key for a keyed chain) MUST report tampering. - -`verify_dag(events) -> errors[]` additionally checks the causal structure: -`seq_id`s are unique, and every `triggered_by` references an existing, -**strictly earlier** `seq_id`. The strictly-earlier rule makes rewind-by- -truncation sound (cutting at seq N never orphans a survivor). - -## 6. Conformance - -[`conformance.json`](./conformance.json) lists vectors in [`vectors/`](./vectors/): - -- **intact** vectors MUST `verify_chain == []` **and** the last event's - `entry_hash` MUST equal the frozen `tip_entry_hash`. This is the cross-impl - oracle — reproduce the tip or you are not conformant. -- **tampered** vectors MUST produce a non-empty error containing the named - `seq`. -- the HMAC vector uses key `"korg-conformance-key"`; verifying it with no key - MUST fail. - -Run the reference harness: `python3 spec/korg-ledger-v1/conformance.py` -(exit 0 = conformant). Regenerate vectors: `python3 spec/korg-ledger-v1/_generate_vectors.py`. - -## 7. v1 scope / non-goals - -- v1 defines integrity (chain) and causal well-formedness (DAG). It does **not** - define event semantics, signatures over the chain *tip* (an Ed25519 signature - over the final `entry_hash` is a v1.1 candidate), or transport. -- Floats are out of scope for v1 canonicalization (korg events don't emit them); - add them under JCS number rules in a future version if needed. diff --git a/crates/korg-registry/tests/conformance/basic-intact.jsonl b/crates/korg-registry/tests/conformance/basic-intact.jsonl deleted file mode 100644 index 5fd77b5..0000000 --- a/crates/korg-registry/tests/conformance/basic-intact.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"schema_version": "1.0", "seq_id": 1, "source_agent": "agent:conformance", "tool_name": "user_prompt", "args": {"prompt": "add a function"}, "result": {}, "success": true, "duration_ms": 0, "prev_hash": "0000000000000000000000000000000000000000000000000000000000000000", "entry_hash": "845344fd0d2830343a5494005420c492eacf3bbf244d671dc9dc1e9ec1f23bb4"} -{"schema_version": "1.0", "seq_id": 2, "source_agent": "agent:conformance", "tool_name": "llm_inference", "args": {"model": "m", "prompt_tokens": 10}, "result": {"completion_tokens": 4}, "success": true, "duration_ms": 12, "triggered_by": 1, "prev_hash": "845344fd0d2830343a5494005420c492eacf3bbf244d671dc9dc1e9ec1f23bb4", "entry_hash": "675d596a62e4a1c24cbbcc3e0ce80612a875b875df72897a2d21afe910e1088c"} -{"schema_version": "1.0", "seq_id": 3, "source_agent": "agent:conformance", "tool_name": "Write", "args": {"path": "mathx.py"}, "result": {"ok": true}, "success": true, "duration_ms": 3, "triggered_by": 2, "prev_hash": "675d596a62e4a1c24cbbcc3e0ce80612a875b875df72897a2d21afe910e1088c", "entry_hash": "7418b9105a664e21078fde881fbd8a5295c49bb384aa62a48d5b544292d910af"} diff --git a/crates/korg-registry/tests/conformance/conformance.json b/crates/korg-registry/tests/conformance/conformance.json deleted file mode 100644 index e0031b7..0000000 --- a/crates/korg-registry/tests/conformance/conformance.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "spec_version": "korg-ledger@v1", - "canonicalization": "JSON, keys sorted by code point, separators (',',':'), non-ASCII \\uXXXX-escaped; preimage = event minus entry_hash", - "hmac": "HMAC-SHA256 over the same preimage when a key is present", - "vectors": [ - { - "file": "basic-intact.jsonl", - "key": null, - "verify": "intact", - "tip_entry_hash": "7418b9105a664e21078fde881fbd8a5295c49bb384aa62a48d5b544292d910af" - }, - { - "file": "hmac-intact.jsonl", - "key": "korg-conformance-key", - "verify": "intact", - "tip_entry_hash": "1b371102038082d3fac965fe21ba357129c61f57fcd93f1a0ffa5e6eaa454e78" - }, - { - "file": "nonbmp-intact.jsonl", - "key": null, - "verify": "intact", - "tip_entry_hash": "045d2d865e89c54340408b309c81d6b0f1367ebf0aa4504f1af66f7f1e4cb590" - }, - { - "file": "tampered-content.jsonl", - "key": null, - "verify": "tampered", - "error_contains": "seq 2" - }, - { - "file": "tampered-deletion.jsonl", - "key": null, - "verify": "tampered", - "error_contains": "seq 3" - } - ] -} diff --git a/crates/korg-registry/tests/conformance/hmac-intact.jsonl b/crates/korg-registry/tests/conformance/hmac-intact.jsonl deleted file mode 100644 index 35344c4..0000000 --- a/crates/korg-registry/tests/conformance/hmac-intact.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"schema_version": "1.0", "seq_id": 1, "source_agent": "agent:conformance", "tool_name": "user_prompt", "args": {"prompt": "add a function"}, "result": {}, "success": true, "duration_ms": 0, "prev_hash": "0000000000000000000000000000000000000000000000000000000000000000", "entry_hash": "0c3524bd96bb2a92f110691a7d26039e1ed457bf272a70a1e32f13f76003267e"} -{"schema_version": "1.0", "seq_id": 2, "source_agent": "agent:conformance", "tool_name": "llm_inference", "args": {"model": "m", "prompt_tokens": 10}, "result": {"completion_tokens": 4}, "success": true, "duration_ms": 12, "triggered_by": 1, "prev_hash": "0c3524bd96bb2a92f110691a7d26039e1ed457bf272a70a1e32f13f76003267e", "entry_hash": "022679afb23ac3eae79ddd4fbdc5242d7b19d10444c6147afadf562bf435dbf8"} -{"schema_version": "1.0", "seq_id": 3, "source_agent": "agent:conformance", "tool_name": "Write", "args": {"path": "mathx.py"}, "result": {"ok": true}, "success": true, "duration_ms": 3, "triggered_by": 2, "prev_hash": "022679afb23ac3eae79ddd4fbdc5242d7b19d10444c6147afadf562bf435dbf8", "entry_hash": "1b371102038082d3fac965fe21ba357129c61f57fcd93f1a0ffa5e6eaa454e78"} diff --git a/crates/korg-registry/tests/conformance/tampered-content.jsonl b/crates/korg-registry/tests/conformance/tampered-content.jsonl deleted file mode 100644 index bc6e2b6..0000000 --- a/crates/korg-registry/tests/conformance/tampered-content.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"schema_version": "1.0", "seq_id": 1, "source_agent": "agent:conformance", "tool_name": "user_prompt", "args": {"prompt": "add a function"}, "result": {}, "success": true, "duration_ms": 0, "prev_hash": "0000000000000000000000000000000000000000000000000000000000000000", "entry_hash": "845344fd0d2830343a5494005420c492eacf3bbf244d671dc9dc1e9ec1f23bb4"} -{"schema_version": "1.0", "seq_id": 2, "source_agent": "agent:conformance", "tool_name": "llm_inference", "args": {"model": "EVIL", "prompt_tokens": 10}, "result": {"completion_tokens": 4}, "success": true, "duration_ms": 12, "triggered_by": 1, "prev_hash": "845344fd0d2830343a5494005420c492eacf3bbf244d671dc9dc1e9ec1f23bb4", "entry_hash": "675d596a62e4a1c24cbbcc3e0ce80612a875b875df72897a2d21afe910e1088c"} -{"schema_version": "1.0", "seq_id": 3, "source_agent": "agent:conformance", "tool_name": "Write", "args": {"path": "mathx.py"}, "result": {"ok": true}, "success": true, "duration_ms": 3, "triggered_by": 2, "prev_hash": "675d596a62e4a1c24cbbcc3e0ce80612a875b875df72897a2d21afe910e1088c", "entry_hash": "7418b9105a664e21078fde881fbd8a5295c49bb384aa62a48d5b544292d910af"} diff --git a/crates/korg-registry/tests/conformance/tampered-deletion.jsonl b/crates/korg-registry/tests/conformance/tampered-deletion.jsonl deleted file mode 100644 index 0914fb0..0000000 --- a/crates/korg-registry/tests/conformance/tampered-deletion.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"schema_version": "1.0", "seq_id": 1, "source_agent": "agent:conformance", "tool_name": "user_prompt", "args": {"prompt": "add a function"}, "result": {}, "success": true, "duration_ms": 0, "prev_hash": "0000000000000000000000000000000000000000000000000000000000000000", "entry_hash": "845344fd0d2830343a5494005420c492eacf3bbf244d671dc9dc1e9ec1f23bb4"} -{"schema_version": "1.0", "seq_id": 3, "source_agent": "agent:conformance", "tool_name": "Write", "args": {"path": "mathx.py"}, "result": {"ok": true}, "success": true, "duration_ms": 3, "triggered_by": 2, "prev_hash": "675d596a62e4a1c24cbbcc3e0ce80612a875b875df72897a2d21afe910e1088c", "entry_hash": "7418b9105a664e21078fde881fbd8a5295c49bb384aa62a48d5b544292d910af"} diff --git a/crates/korg-server/Cargo.toml b/crates/korg-server/Cargo.toml index 3d3734f..baee375 100644 --- a/crates/korg-server/Cargo.toml +++ b/crates/korg-server/Cargo.toml @@ -9,6 +9,7 @@ license = "MIT OR Apache-2.0" korg-core = { path = "../korg-core" } korg-auth = { path = "../korg-auth" } korg-registry = { path = "../korg-registry" } +korg-ledger = { path = "../korg-ledger", features = ["redact"] } korg-llm = { path = "../korg-llm" } korg-runtime = { path = "../korg-runtime" } serde = { workspace = true } diff --git a/crates/korg-server/src/lib.rs b/crates/korg-server/src/lib.rs index 3cfcb02..73e3f4d 100644 --- a/crates/korg-server/src/lib.rs +++ b/crates/korg-server/src/lib.rs @@ -847,11 +847,17 @@ async fn agent_tool_call_handler( use std::collections::BTreeMap; use uuid::Uuid; + // Redaction at persistence (spec/korg-ledger-v1/REDACTION.md): the payload is + // redacted BEFORE it enters the journal, so the hash chain only ever covers + // redacted bytes. Redacting after append would be indistinguishable from + // tampering. + let (args, _) = korg_ledger::redact::redact(&req.args); + let (result, _) = korg_ledger::redact::redact(&req.result); let event = CapabilityEvent::AgentToolCall { source_agent: req.source_agent.clone(), tool_name: req.tool_name, - args: req.args, - result: req.result, + args, + result, payload_refs: req.payload_refs, success: req.success, duration_ms: req.duration_ms, @@ -860,6 +866,25 @@ async fn agent_tool_call_handler( let mut resolver = state.capability_resolver.lock().await; + // Causal-parent contract (spec §9): triggered_by must name an existing, + // strictly-earlier seq_id. Persisting a dangling or forward link would write + // a journal that every verifier now rejects, so refuse it up front. + if let Some(tb) = req.triggered_by { + let next_seq = resolver.journal.last_seq_id + 1; + let exists = resolver.journal.events.iter().any(|e| e.seq_id == tb); + if tb >= next_seq || !exists { + return ( + StatusCode::UNPROCESSABLE_ENTITY, + Json(serde_json::json!({ + "error": format!( + "triggered_by {tb} must reference an existing, strictly-earlier seq_id (next seq_id is {next_seq})" + ) + })), + ) + .into_response(); + } + } + // ALWAYS use append_with_metadata for external agent events. // // The standard append() auto-sets triggered_by to the previous journal @@ -913,7 +938,10 @@ async fn agent_tool_call_handler( retry_count: 0, tier: EventTier::Telemetry, span_id: None, - tags: BTreeMap::new(), + tags: BTreeMap::from([( + korg_ledger::redact::TAG.to_string(), + korg_ledger::redact::POLICY.to_string(), + )]), triggered_by: req.triggered_by, // preserved exactly — None means root }; @@ -2972,6 +3000,127 @@ mod tests { let _ = std::fs::remove_dir_all(&temp_dir); } + fn tool_call_test_state() -> ( + Arc, + Arc>, + std::path::PathBuf, + ) { + let (broadcaster_tx, _) = tokio::sync::broadcast::channel(16); + let (feedback_tx, _) = tokio::sync::mpsc::channel(16); + let temp_dir = std::env::temp_dir().join(format!("korg_test_{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&temp_dir).unwrap(); + let journal = CapabilityJournal::new( + temp_dir.join("journal.json"), + temp_dir.join("snapshot.json"), + 10, + temp_dir.join("lock.lock"), + ); + let resolver = Arc::new(TokioMutex::new(CapabilityResolver::new( + std::collections::HashMap::new(), + journal, + ))); + let auth_state = Arc::new(korg_auth::AuthState::new(korg_auth::AuthConfig { + base_url: "http://localhost:8080".to_string(), + codex_client_id: "mock-codex-client-id".to_string(), + codex_client_secret: "mock-codex-client-secret".to_string(), + anthropic_client_id: "mock-anthropic-client-id".to_string(), + anthropic_client_secret: "mock-anthropic-client-secret".to_string(), + token_store_path: temp_dir.join("auth.json"), + })); + let app_state = Arc::new(AppState { + broadcaster: broadcaster_tx, + feedback_tx: TokioMutex::new(Some(feedback_tx)), + capability_resolver: resolver.clone(), + runtime_coordinator: Arc::new(StdMutex::new(None)), + auth: auth_state, + }); + (app_state, resolver, temp_dir) + } + + fn tool_call( + tool: &str, + args: serde_json::Value, + triggered_by: Option, + ) -> AgentToolCallRequest { + AgentToolCallRequest { + source_agent: "agent:claude-code@2.1.0".to_string(), + tool_name: tool.to_string(), + args, + result: serde_json::json!({}), + payload_refs: vec![], + success: true, + duration_ms: 1, + triggered_by, + } + } + + #[tokio::test] + async fn test_agent_tool_call_redacts_before_the_journal_and_tags_the_policy() { + let (state, resolver, dir) = tool_call_test_state(); + let secret = "sk-proj-THISISASECRETVALUE0123456789"; + let req = tool_call( + "Bash", + serde_json::json!({"command": format!("export OPENAI_API_KEY={secret}"), "api_key": "k"}), + None, + ); + let resp = agent_tool_call_handler(State(state), Json(req)) + .await + .into_response(); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + + let r = resolver.lock().await; + let ev = r.journal.events.last().unwrap(); + let line = serde_json::to_string(ev).unwrap(); + assert!(!line.contains(secret), "secret reached the journal: {line}"); + assert!(line.contains("export OPENAI_API_KEY=[REDACTED:env_assignment]")); + assert!(line.contains("\"api_key\":\"[REDACTED:sensitive_key]\"")); + assert_eq!( + ev.metadata.tags.get("korg.redaction").map(String::as_str), + Some("korg-redact@v1") + ); + // the persisted record is clean under the same lint verifiers run + let v = serde_json::to_value(ev).unwrap(); + assert!(korg_ledger::redact::verify_redaction(&[v]).is_empty()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn test_agent_tool_call_rejects_dangling_or_forward_triggered_by() { + let (state, resolver, dir) = tool_call_test_state(); + let ok = agent_tool_call_handler( + State(state.clone()), + Json(tool_call("user_prompt", serde_json::json!({}), None)), + ) + .await + .into_response(); + assert_eq!(ok.status(), axum::http::StatusCode::OK); + let root = resolver.lock().await.journal.last_seq_id; + + for bad in [root + 1, root + 50] { + let resp = agent_tool_call_handler( + State(state.clone()), + Json(tool_call("Read", serde_json::json!({}), Some(bad))), + ) + .await + .into_response(); + assert_eq!(resp.status(), axum::http::StatusCode::UNPROCESSABLE_ENTITY); + } + assert_eq!( + resolver.lock().await.journal.last_seq_id, + root, + "rejected calls must not append" + ); + + let child = agent_tool_call_handler( + State(state), + Json(tool_call("Read", serde_json::json!({}), Some(root))), + ) + .await + .into_response(); + assert_eq!(child.status(), axum::http::StatusCode::OK); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn test_oauth_pkce_csrfs_distinct() { let config = korg_auth::AuthConfig { diff --git a/crates/korg-verify/Cargo.toml b/crates/korg-verify/Cargo.toml index 86b2fe1..7a7ed4c 100644 --- a/crates/korg-verify/Cargo.toml +++ b/crates/korg-verify/Cargo.toml @@ -15,7 +15,7 @@ name = "korg-verify" path = "src/main.rs" [dependencies] -korg-ledger = { path = "../korg-ledger", version = "0.2.0", features = ["signing"] } +korg-ledger = { path = "../korg-ledger", version = "0.2.0", features = ["signing", "redact"] } serde_json = { workspace = true } ed25519-dalek = "2" hex = "0.4" diff --git a/crates/korg-verify/README.md b/crates/korg-verify/README.md index b7f7ba1..7aa0a88 100644 --- a/crates/korg-verify/README.md +++ b/crates/korg-verify/README.md @@ -3,7 +3,7 @@ A standalone, dependency-light **verifier** for korg receipts and journals — no network, no Python. ``` -korg-verify [--key ] [--pubkey ] [--pin-event-pubkey ] [--anchors ] [--json] +korg-verify [--key ] [--pubkey ] [--pin-event-pubkey ] [--anchors ] [--require-redacted] [--json] ``` Exit code: `0` valid · `1` invalid/tampered · `2` usage/parse error. @@ -42,4 +42,4 @@ cargo build --release -p korg-verify # → target/release/korg-verify ## Tests -`cargo test -p korg-verify` runs against the shared `crates/korg-registry/tests/conformance` vectors (intact, HMAC-keyed, non-BMP unicode, and tampered cases) plus a real receipt minted by `korgex receipt --sign` — cross-implementation proof that Rust re-derives the chain and verifies the Python-produced Ed25519 signature. +`cargo test -p korg-verify` runs against the canonical `spec/korg-ledger-v1` vectors (intact, HMAC-keyed, non-BMP unicode, tampered, flat + nested causal-DAG cases with exact errors, and the korg-redact@v1 cases) plus a real receipt minted by `korgex receipt --sign` — cross-implementation proof that Rust re-derives the chain and verifies the Python-produced Ed25519 signature. diff --git a/crates/korg-verify/src/lib.rs b/crates/korg-verify/src/lib.rs index d326dae..0085263 100644 --- a/crates/korg-verify/src/lib.rs +++ b/crates/korg-verify/src/lib.rs @@ -15,6 +15,7 @@ //! that the key maps to a real-world identity (the relying party pins that — see //! `--pubkey`). +pub use korg_ledger::redact::verify_redaction; use korg_ledger::{verify_chain, verify_dag}; use serde_json::Value; @@ -48,6 +49,9 @@ pub struct Verdict { /// `None` for receipts/journals; `Some(true/false)` for a korgcert@v1 — whether /// the embedded summary byte-matches the summary re-derived from the events. pub summary_ok: Option, + /// `None` unless the redaction lint was requested (`--require-redacted`); + /// `Some(true/false)` = every event is clean under korg-redact@v1. + pub redaction_ok: Option, pub errors: Vec, } @@ -115,6 +119,7 @@ pub fn verify_journal(events: &[Value], key: Option<&[u8]>) -> Verdict { event_sigs_ok: None, anchors_ok: None, summary_ok: None, + redaction_ok: None, errors, } } @@ -171,6 +176,21 @@ pub fn verify_journal_extended( v } +/// Apply the korg-redact@v1 cleanliness lint (spec `REDACTION.md`) to an +/// existing verdict: every persisted event must already be redacted. A finding +/// means a secret reached the ledger in plaintext — a producer contract breach +/// that no later edit can repair without breaking the chain. +pub fn require_redacted(mut v: Verdict, events: &[Value]) -> Verdict { + let errs = verify_redaction(events); + let ok = errs.is_empty(); + v.redaction_ok = Some(ok); + v.errors.extend(errs); + if !ok { + v.valid = false; + } + v +} + /// Verify a receipt object: embedded events (chain + DAG), the recorded tip matches /// the chain head, and — if signed — the Ed25519 signature is valid for that tip. /// @@ -267,6 +287,7 @@ pub fn verify_receipt(receipt: &Value, key: Option<&[u8]>, pin_pubkey: Option<&s event_sigs_ok: None, anchors_ok: None, summary_ok: None, + redaction_ok: None, errors, } } @@ -474,6 +495,7 @@ pub fn verify_korgcert(envelope: &Value, pin_pubkey: Option<&str>) -> Verdict { event_sigs_ok: None, anchors_ok, summary_ok: Some(summary_ok), + redaction_ok: None, errors, } } diff --git a/crates/korg-verify/src/main.rs b/crates/korg-verify/src/main.rs index c6270b3..df0ab5e 100644 --- a/crates/korg-verify/src/main.rs +++ b/crates/korg-verify/src/main.rs @@ -7,7 +7,7 @@ const HELP: &str = "\ korg-verify — verify a korg receipt or journal (hash chain + causal DAG + Ed25519) USAGE: - korg-verify [--key ] [--pubkey ] [--pin-event-pubkey ] [--anchors ] [--json] + korg-verify [--key ] [--pubkey ] [--pin-event-pubkey ] [--anchors ] [--require-redacted] [--json] ARGS: a Certificate (korgcert@v1), a korg receipt (.json), or a journal (.jsonl / JSON array) @@ -17,6 +17,7 @@ OPTIONS: --pubkey pin the expected signer — the receipt-tip signer or the Certificate issuer --pin-event-pubkey require every event's per-event Ed25519 event_sig to verify under this key --anchors verify an anchors.jsonl sidecar (structural: entry_hash ↔ chain) + --require-redacted fail if any event holds a secret korg-redact@v1 would have redacted --json machine-readable verdict -h, --help show this help @@ -31,6 +32,7 @@ fn main() -> ExitCode { let mut pin_event_pubkey: Option = None; let mut anchors_path: Option = None; let mut json_out = false; + let mut require_redacted = false; let mut i = 0; while i < args.len() { @@ -55,6 +57,10 @@ fn main() -> ExitCode { json_out = true; i += 1; } + "--require-redacted" => { + require_redacted = true; + i += 1; + } "-h" | "--help" => { print!("{HELP}"); return ExitCode::from(2); @@ -123,6 +129,20 @@ fn main() -> ExitCode { } }; + // The redaction lint runs over whatever events the artifact embeds (journal + // lines, or a receipt's / Certificate's `events`). + let verdict = if require_redacted { + let events = match serde_json::from_str::(&text) { + Ok(v) if v.get("events").is_some_and(|e| e.is_array()) => { + v["events"].as_array().cloned().unwrap_or_default() + } + _ => korg_verify::load_events(&text).unwrap_or_default(), + }; + korg_verify::require_redacted(verdict, &events) + } else { + verdict + }; + if json_out { let out = serde_json::json!({ "valid": verdict.valid, @@ -136,6 +156,7 @@ fn main() -> ExitCode { "event_sigs_ok": verdict.event_sigs_ok, "anchors_ok": verdict.anchors_ok, "summary_ok": verdict.summary_ok, + "redaction_ok": verdict.redaction_ok, "errors": verdict.errors, }); println!("{out}"); diff --git a/crates/korg-verify/tests/conformance.rs b/crates/korg-verify/tests/conformance.rs index 554b7f7..45ac799 100644 --- a/crates/korg-verify/tests/conformance.rs +++ b/crates/korg-verify/tests/conformance.rs @@ -1,24 +1,98 @@ -//! korg-verify against the SHARED frozen korg-ledger@v1 vectors — the same oracle -//! the Python and JS implementations are checked against. A green run here means a -//! third independent implementation reproduces the chain, not just a second one. +//! korg-verify against the SHARED frozen korg-ledger@v1 vectors in the canonical +//! spec directory (`spec/korg-ledger-v1/conformance.json` + `vectors/`) — the +//! same manifest the Python and JS implementations are checked against. A green +//! run here means a third independent implementation reproduces the chain, the +//! exact causal-DAG errors (§9), and the redaction lint (REDACTION.md). -use korg_verify::verify_text; +use korg_verify::{verify_journal, verify_redaction, verify_text}; +use serde_json::Value; use std::path::PathBuf; -fn read(name: &str) -> String { - let p = PathBuf::from(concat!( +fn spec_dir() -> PathBuf { + PathBuf::from(concat!( env!("CARGO_MANIFEST_DIR"), - "/../korg-registry/tests/conformance" + "/../../spec/korg-ledger-v1" )) - .join(name); +} + +fn read(name: &str) -> String { + let p = spec_dir().join("vectors").join(name); std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) } +fn manifest() -> Value { + serde_json::from_str(&std::fs::read_to_string(spec_dir().join("conformance.json")).unwrap()) + .unwrap() +} + +fn strings(v: &Value) -> Vec { + v.as_array() + .map(|a| a.iter().map(|s| s.as_str().unwrap().to_string()).collect()) + .unwrap_or_default() +} + +#[test] +fn every_manifest_vector_reproduces_chain_dag_and_redaction() { + let m = manifest(); + assert_eq!(m["spec_version"], "korg-ledger@v1"); + let vectors = m["vectors"].as_array().unwrap(); + assert!(vectors.len() >= 11, "manifest lost vectors"); + for v in vectors { + let file = v["file"].as_str().unwrap(); + let events = korg_verify::load_events(&read(file)).unwrap(); + let key = v["key"].as_str().map(|k| k.as_bytes().to_vec()); + let verdict = verify_journal(&events, key.as_deref()); + let chain_errs = korg_ledger::verify_chain(&events, key.as_deref()); + match v["verify"].as_str().unwrap() { + "intact" => { + assert!(chain_errs.is_empty(), "{file}: {chain_errs:?}"); + let tip = korg_ledger::chain_hash(events.last().unwrap(), key.as_deref()); + assert_eq!(tip, v["tip_entry_hash"].as_str().unwrap(), "{file}: tip"); + } + _ => { + let needle = v["error_contains"].as_str().unwrap(); + assert!( + chain_errs.iter().any(|e| e.contains(needle)), + "{file}: {chain_errs:?}" + ); + } + } + let want_dag = if v["dag"] == "broken" { + strings(&v["dag_errors"]) + } else { + vec![] + }; + assert_eq!( + korg_ledger::verify_dag(&events), + want_dag, + "{file}: dag errors" + ); + assert_eq!(verdict.dag_ok, want_dag.is_empty(), "{file}: dag_ok"); + if let Some(r) = v["redaction"].as_str() { + let want = if r == "unclean" { + strings(&v["redaction_errors"]) + } else { + vec![] + }; + assert_eq!(verify_redaction(&events), want, "{file}: redaction lint"); + } + } +} + #[test] -fn basic_intact_is_valid() { - let v = verify_text(&read("basic-intact.jsonl"), None, None).unwrap(); - assert!(v.valid, "errors: {:?}", v.errors); - assert!(v.chain_ok && v.dag_ok); +fn manifest_lists_every_vector_on_disk() { + let listed: std::collections::BTreeSet = manifest()["vectors"] + .as_array() + .unwrap() + .iter() + .map(|v| v["file"].as_str().unwrap().to_string()) + .collect(); + let on_disk: std::collections::BTreeSet = std::fs::read_dir(spec_dir().join("vectors")) + .unwrap() + .filter_map(|e| e.ok()?.file_name().into_string().ok()) + .filter(|n| n.ends_with(".jsonl")) + .collect(); + assert_eq!(listed, on_disk); } #[test] @@ -27,8 +101,7 @@ fn hmac_intact_valid_only_with_the_key() { assert!( verify_text(&txt, Some(b"korg-conformance-key"), None) .unwrap() - .valid, - "should be valid with the right HMAC key" + .valid ); assert!( !verify_text(&txt, None, None).unwrap().valid, @@ -37,30 +110,41 @@ fn hmac_intact_valid_only_with_the_key() { } #[test] -fn nonbmp_intact_is_valid() { - // astral-plane code points — exercises the \uXXXX surrogate-pair canonicalization - let v = verify_text(&read("nonbmp-intact.jsonl"), None, None).unwrap(); - assert!(v.valid, "errors: {:?}", v.errors); -} - -#[test] -fn tampered_content_flags_seq_2() { - let v = verify_text(&read("tampered-content.jsonl"), None, None).unwrap(); - assert!(!v.valid); - assert!( - v.errors.iter().any(|e| e.contains("seq 2")), - "{:?}", - v.errors - ); -} - -#[test] -fn tampered_deletion_flags_seq_3() { - let v = verify_text(&read("tampered-deletion.jsonl"), None, None).unwrap(); - assert!(!v.valid); - assert!( - v.errors.iter().any(|e| e.contains("seq 3")), - "{:?}", - v.errors - ); +fn shared_redaction_cases() { + use korg_ledger::redact::{finding_order, redact}; + let cases: Value = serde_json::from_str( + &std::fs::read_to_string(spec_dir().join("vectors/redaction/korg-redact-v1.json")).unwrap(), + ) + .unwrap(); + assert_eq!(cases["policy"], "korg-redact@v1"); + for c in cases["cases"].as_array().unwrap() { + let name = c["name"].as_str().unwrap(); + let want = if c["output"].is_null() { + &c["input"] + } else { + &c["output"] + }; + let (got, mut findings) = redact(&c["input"]); + assert_eq!(&got, want, "{name}: output"); + findings.sort_by(finding_order); + let got_f: Vec<(String, String)> = findings + .iter() + .map(|f| (f.path.clone(), f.rule.to_string())) + .collect(); + let want_f: Vec<(String, String)> = c["findings"] + .as_array() + .unwrap() + .iter() + .map(|f| { + ( + f["path"].as_str().unwrap().to_string(), + f["rule"].as_str().unwrap().to_string(), + ) + }) + .collect(); + assert_eq!(got_f, want_f, "{name}: findings"); + let (again, more) = redact(&got); + assert_eq!(again, got, "{name}: idempotent"); + assert!(more.is_empty(), "{name}: output is clean"); + } } diff --git a/mcp_server.py b/mcp_server.py index 9bbf0e0..e07b23c 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -791,7 +791,7 @@ def _resource_stats_integrity(_query: dict[str, str]) -> dict: ] # §1.1 actor convention prefixes - valid_prefixes = ("agent:", "human:", "korg:", "mcp:") + valid_prefixes = ("agent:", "human:", "korg:", "mcp:", "import:") actor_counts: dict[str, int] = {} violations: list[str] = [] schema_versions: dict[str, int] = {} diff --git a/spec/korg-ledger-v1/IMPORT.md b/spec/korg-ledger-v1/IMPORT.md new file mode 100644 index 0000000..0a7cc03 --- /dev/null +++ b/spec/korg-ledger-v1/IMPORT.md @@ -0,0 +1,154 @@ +# korg-import@v1 — Import Envelopes + +**Status:** NORMATIVE (korg-ledger@v1 revision 1.1, [SPEC.md §11](./SPEC.md#11-import-envelopes-revision-11)) · +**Implementation:** [`korg_ledger.imports`](../../adapters/korg-ledger-py/src/korg_ledger/imports.py) · **Examples:** [`imports/`](./imports/) + +A flight recorder sometimes needs evidence it did not observe: what the proxy in +front of the agent saw, or what the secret-holding sidecar did. That evidence +comes from tools with their own formats and their own secrets. An **import +envelope** is the only door: a strict, versioned, allowlisted container that +carries *metadata about* those systems and never their secret values. + +## 1. Envelope + +```json +{ + "schema": "korg-import@v1", + "profile": "okto.diagnostics@v1", + "source": {"name": "okto", "version": "0.1.0"}, + "exported_at_ms": 1781395260000, + "dropped": 0, + "records": [ {"kind": "session", "...": "..."} ] +} +``` + +| Field | Required | Rule | +|---|---|---| +| `schema` | yes | exactly `"korg-import@v1"` | +| `profile` | yes | a registered profile id (§3) | +| `source` | yes | exactly `{name, version}`; `name` MUST equal the profile's source; `version` matches `[0-9A-Za-z][0-9A-Za-z.+-]{0,31}` | +| `exported_at_ms` | no | non-negative ledger integer | +| `dropped` | no | non-negative integer: source items the exporter could not project and left out | +| `records` | yes | 1–10 000 objects, each with a `kind` registered by the profile | + +No other envelope field is allowed. + +## 2. Validation (importer obligations) + +An importer MUST reject the **whole envelope**, appending nothing, if any of +these holds (all violations are reported, values are never echoed in errors): + +1. an unknown envelope field, schema, profile, or source mismatch; +2. a record whose `kind` the profile does not register; +3. a record field the kind does not allow — **allowlist, not denylist**. Fields + the profile names as *forbidden* get a specific error ("may carry secrets or + raw traffic"), but any unlisted field is rejected the same way; +4. a missing required field, or a value of the wrong type. Types are: ledger + integer (non-negative unless a range is given), boolean, enum, **id** + (`[A-Za-z0-9][A-Za-z0-9._:-]{0,127}`), **slug** (`[a-z0-9][a-z0-9_.-]{0,63}`), + and a slug→integer map (≤16 entries). **There is no free-text type**; +5. any record that is not redaction-clean under + [korg-redact@v1](./REDACTION.md) (a second gate behind the allowlist). + +Floats are never allowed (ledger numbers are integers): a fractional source +value is converted by the exporter (`duration_ms` rounded, Okto `confidence` +0–1 → `confidence_bp` 0–10000). + +## 3. Mapping into the ledger + +A valid envelope becomes `1 + len(records)` events, appended in order: + +- **batch event** — `tool_name: "korg.import"`, `args: {profile, record_count, + envelope_sha256, [exported_at_ms], [dropped]}`, `triggered_by` = the caller's + parent (or root). `envelope_sha256` is SHA-256 over the canonical (§2) + envelope bytes, so anyone holding the sanitized envelope can prove it is + exactly what was imported. +- **one event per record** — `tool_name: "."` (e.g. + `okto.anomaly`, `ningen-shikkaku.trigger`), `args` = the record minus `kind`, + `result: {import_index: i}`, `triggered_by` = the batch event's `seq_id`. + +Every imported event has `source_agent = "import:@"` — a fifth +actor prefix alongside `agent:`, `human:`, `korg:`, `mcp:` — and recorder +`actor_id = "korg:import"`. Imported events are **attested by the importer, not +observed by korg**: the chain proves they were not altered after import, not +that the source reported truthfully. + +## 4. Profile `okto.diagnostics@v1` + +Source: [Okto](https://github.com/New1Direction/okto), the self-healing local +proxy for agent traffic (`source.name = "okto"`). Okto's own captures hold +request/response headers (including `authorization`, `x-api-key`, `cookie`), +full prompt bodies, raw stream bytes, and anomaly descriptions quoting payload +bytes — none of which may cross. + +| kind | required | optional | +|---|---|---| +| `session` | `session_id` id, `client` enum¹ | `at_ms`, `tls_established` bool, `synthetic` bool, `provider_vendor` slug, `provider_product` slug, `turn_count`, `anomaly_count`, `mutation_count` | +| `turn` | `session_id`, `turn_index` | `at_ms`, `status_code` 100–599, `duration_ms`, `chunk_count`, `anomaly_count`, `mutations_applied` (count), `token_usage` slug→int map | +| `anomaly` | `session_id`, `turn_index`, `chunk_index`, `anomaly_type` enum² | `at_ms`, `repaired` bool | +| `mutation` | `mutation_id` id, `scope` enum³, `patch_type` enum⁴ | `at_ms`, `session_id`, `client`, `confidence_bp` 0–10000, `reinforcement_count` | +| `replay` | `session_id`, `resolved` bool | `at_ms`, `total_anomalies`, `total_mutations_applied` | + +¹ `claude gemini codex agy kimi kimi-code cursor opencode pi hermes unknown` · +² `bare_json_without_data_prefix malformed_json_in_sse_frame empty_response_body line_ending_mismatch protocol_anomaly other` · +³ `request_header request_body response_header response_body stream_chunk` · +⁴ `literal regex_sub json_patch header_set` + +**Forbidden (named):** `request_headers response_headers headers request_body +response_body body raw_hex expected_hex patched_hex pattern_hex patch_hex +description parsed chunks stream_chunks path url connect_target hostname +node_id`. `connect_target`/`hostname`/`path` are excluded because they can name +internal infrastructure or carry query-string tokens. + +**Exporters** (`korg_ledger.imports`, projection only — they never read a +forbidden field): `okto_session_to_envelope(Session)` for +`~/.okto/sessions/.jsonl`, `okto_mutation_record(Mutation)` for +`~/.okto/genome.jsonl`. CLI: `python -m korg_ledger envelope okto-session FILE +--source-version 0.1.0`. + +## 5. Profile `ningen-shikkaku.lifecycle@v1` + +Source: [ningen-shikkaku](https://github.com/New1Direction/ningen-shikkaku) — +the `dazai` session-bound dead-man's switch and `motokano` one-shot MCP secret +server (`source.name = "ningen-shikkaku"`). Its secrets live in locked memory and +are returned by `motokano` tool calls; its tool specs (`--tool …value=…`) and +error lines can echo them. Only lifecycle **metadata** may cross. + +| kind | required | optional | +|---|---|---| +| `daemon_start` | `mode` (`armed` \| `dry_run`) | `at_ms`, `pid`, `grace_ms`, `ping_timeout_ms`, `seccomp` bool | +| `buffers` | `allocated` | `at_ms`, `locked` | +| `status` | `alive` bool, `armed` bool | `at_ms`, `grace_seconds`, `registered_pids` | +| `armed` | `via` (`flag` \| `control` \| `mcp`) | `at_ms` | +| `trigger` | `reason`⁵, `mode` (`armed` \| `dry_run`) | `at_ms`, `severity` (`graceful` \| `hard`) | +| `cancelled` | `reason` (`client_reconnected` \| `client_cancel` \| `other`) | `at_ms` | +| `panic` | `reason`⁵ | `at_ms` | +| `wiped` | `buffers` | `at_ms` | +| `pid_registered` | `pid` | `at_ms`, `total` | +| `pid_unregistered` | `pid` | `at_ms` | +| `pid_killed` | `pid`, `outcome` (`sent` \| `already_gone`) | `at_ms` | +| `shutdown` | `how` (`quit` \| `signal`) | `at_ms` | +| `exit_condition_fired` | `counter_mode` (`calls` \| `session` \| `either`) | `at_ms`, `calls` | + +⁵ `heartbeat_dropped ping_deadline_missed sigusr1 sigusr2 grace_expired other` +(mapped from the daemon's literal reason strings, e.g. `"heartbeat connection +dropped"` → `heartbeat_dropped`). + +**Forbidden (named):** `value secret secret_value plaintext data bytes cmd +command spec tool_spec stdout stderr output argv args env message reply +buffer_address address`. + +**Exporter:** `dazai_log_to_envelope(lines)` recognises only the fixed `[dazai]` +lifecycle lines (Rust and Python-reference formats), extracts their numeric and +enum parts, and **drops every other line** — including `error: …` and +`motokano: parsing --tool …` lines that can echo a secret — counting them in +`dropped`. CLI: `python -m korg_ledger envelope dazai-log FILE --source-version +0.1.0`. + +## 6. Adding a profile + +A new profile is a spec change: add its table here, its allowlist to +`korg_ledger.imports.PROFILES`, a valid and a rejected example under +[`imports/`](./imports/), and tests. Profiles are versioned independently +(`.@vN`); widening an existing profile's allowlist requires a new +version. diff --git a/spec/korg-ledger-v1/MIGRATION.md b/spec/korg-ledger-v1/MIGRATION.md new file mode 100644 index 0000000..ea9b3ae --- /dev/null +++ b/spec/korg-ledger-v1/MIGRATION.md @@ -0,0 +1,108 @@ +# Migrating to korg-ledger@v1 revision 1.1 + +Revision 1.1 makes `spec/korg-ledger-v1/` the single authoritative contract for +korg as an agent flight recorder. **No hash rule changed**: every v1.0 vector +reproduces its frozen tip, and every existing journal hashes exactly as before. +What changed is what verifiers check, what producers write, and where the +vectors live. + +## At a glance + +| You are… | What changes | Action | +|---|---|---| +| verifying journals | causal links are now read from nested `metadata.triggered_by` too, and malformed links are errors | re-verify; see [§1](#1-verifiers-nested-causal-links-are-now-checked) | +| writing with `korg_ledger.LedgerWriter` | redaction on by default; `metadata.tags` gains `korg.redaction`; non-integer `triggered_by` raises | usually none; see [§2](#2-python-producer-ledgerwriter) | +| posting to `POST /api/agent/tool-call` | `args`/`result` redacted; bad `triggered_by` → **422** | handle 422; see [§3](#3-http--mcp-producers-korg-server-korg-bridge) | +| running `korg-verify` | new `--require-redacted`; JSON verdict gains `redaction_ok` | optional | +| reading the vectors | one copy in `spec/korg-ledger-v1/vectors/`; manifest gains `dag`, `dag_errors`, `redaction*`, `revision` | repoint paths; see [§4](#4-conformance-vectors) | +| importing Okto / ningen-shikkaku data | new `korg-import@v1` envelopes | see [`IMPORT.md`](./IMPORT.md) | + +## 1. Verifiers: nested causal links are now checked + +Before 1.1, the Rust, Python, and JS `verify_dag` read `triggered_by` only at the +top level. Every production writer stores it under `metadata.triggered_by`, so +for real journals the DAG check was effectively uniqueness-only: a forward or +dangling link passed. They also disagreed on odd values (`true` counted as 1 in +Python only; `1.5` was checked in JS only). + +From 1.1 (SPEC §9) all three resolve the parent from either shape, require a +ledger integer, reject top-level/metadata conflicts, and emit identical error +strings. **A journal that verified before can now fail** — only if it carries a +malformed or impossible link, which 1.0 already forbade in prose. + +Check an existing journal: + +```sh +korg-verify path/to/ledger.jsonl # Rust +node spec/korg-ledger-v1/js/verify.mjs path/to/ledger.jsonl +PYTHONPATH=adapters/korg-ledger-py/src python3 -m korg_ledger lint path/to/ledger.jsonl +``` + +If it now reports `seq N: triggered_by T does not exist` / `is not strictly +earlier`, the producer recorded a bad link (the HTTP endpoint did not validate +links before 1.1). The chain is still intact — the record is authentic, the link +is wrong. Do not edit it (that breaks the chain); keep it as evidence, and fix +the producer. Error-string consumers: Rust previously printed `seq Some(3)` / +`seq None`; all implementations now print `seq 3` / `seq ?`. + +## 2. Python producer (`LedgerWriter`) + +- **Redaction at persistence is on by default** (`redaction="korg-redact@v1"`). + Each record is redacted before it is hashed and written, and + `metadata.tags["korg.redaction"] = "korg-redact@v1"` is added. Payloads with + credential-shaped values are stored with `[REDACTED:]` markers; nothing + else changes. Pass `redaction=None` only for byte-exact fixture replay — those + records carry no tag and fail `--require-redacted` if they hold a secret. +- `triggered_by` must be an `int` (not `bool`, `str`, or `float`); otherwise + `CausalityError` (a `ValueError`) — the Claude Code hook already drops such an + event instead of aborting. +- New: `korg_ledger.redact/scan/verify_redaction`, `causal_parent`, `json_int`, + `korg_ledger.imports`, and the `python -m korg_ledger` CLI. + +## 3. HTTP / MCP producers (korg-server, korg-bridge) + +- `POST /api/agent/tool-call` redacts `args` and `result` before the journal + sees them and tags the event, exactly like `LedgerWriter`. +- A `triggered_by` that does not name an existing, strictly-earlier `seq_id` is + now refused with **422 Unprocessable Entity** and nothing is appended (it used + to be persisted silently, producing a journal that 1.1 verifiers reject). + `mcp_server.py` already surfaces non-2xx responses as tool errors. The + korg-bridge (`append*`) already raised `CausalityError`; it now also redacts. +- `mcp_server.py`'s actor check accepts the new `import:` prefix. + +## 4. Conformance vectors + +- The duplicate copy under `crates/korg-registry/tests/conformance/` (an older + SPEC.md, a manifest with an extra vector) is **removed**. Rust tests read + `spec/korg-ledger-v1/` directly; `nonbmp-intact.jsonl` moved there and is now + checked by Python and JS too. +- New vectors: `nested-intact`, `nested-dag-broken`, `dag-malformed`, + `mixed-shape-intact`, `redacted-intact`, `unredacted-intact`, plus the shared + redaction cases in `vectors/redaction/korg-redact-v1.json`. +- Manifest additions (old harnesses ignore them): `revision`, per-vector `dag` + / `dag_errors`, `redaction` / `redaction_errors`, top-level `causal_parent`, + `redaction_cases`. Existing fields are unchanged. +- `tools/generate_vectors.py` regenerates every vector deterministically and + asserts the v1.0 tips; CI runs it in `--check` mode, plus a manifest ↔ disk + completeness check in every harness. + +## Redacting an existing journal + +A journal that already contains a secret cannot be fixed in place (REDACTION.md +R6). Make a redacted **copy as a new chain**, keep both tips: + +```sh +PYTHONPATH=adapters/korg-ledger-py/src \ + python3 -m korg_ledger rechain-redacted old.jsonl redacted.jsonl [--key K] +# wrote N redacted events to redacted.jsonl +# old tip +# new tip +``` + +The copy preserves `seq_id`s and causal links, declares the policy on every +record, drops `event_sig` (it signed the unredacted bytes — re-sign if needed), +and verifies clean under `korg-verify --require-redacted`. Then rotate the +leaked credential: redaction limits future exposure, it does not undo past +exposure. Anchor or publish the old tip next to the new one so the copy's +provenance stays auditable; delete the original only under your retention +policy. diff --git a/spec/korg-ledger-v1/README.md b/spec/korg-ledger-v1/README.md index 38063fa..a922290 100644 --- a/spec/korg-ledger-v1/README.md +++ b/spec/korg-ledger-v1/README.md @@ -1,6 +1,6 @@ # korg-ledger@v1 — an open standard for tamper-evident agent ledgers -**Status:** FROZEN · **Canonical home:** this directory (`korg/spec/korg-ledger-v1/`) is the normative source; all implementations conform to it. +**Status:** AUTHORITATIVE · revision 1.1 · **Canonical home:** this directory (`korg/spec/korg-ledger-v1/`) is the normative source for korg as an agent flight recorder; all implementations, and every other korg document, conform to it. Upgrading? Read [`MIGRATION.md`](./MIGRATION.md). > **Try it live (zero install, sends nothing):** **[Certificate verifier](https://new1direction.github.io/korg/web/seal.html)** · **[ledger verifier](https://new1direction.github.io/korg/web/index.html)** · **[time-travel explorer](https://new1direction.github.io/korg/web/explore.html)** · **[home](https://new1direction.github.io/korg/)** @@ -19,9 +19,13 @@ the key), not merely tamper-evident. | File | Role | |---|---| -| [`SPEC.md`](./SPEC.md) | the normative specification (canonicalization, preimage, chaining, HMAC, verify + DAG algorithms) | +| [`SPEC.md`](./SPEC.md) | the normative specification (canonicalization, preimage, chaining, HMAC, verify + DAG algorithms, flat vs nested journal shapes and `triggered_by` resolution §9) | +| [`REDACTION.md`](./REDACTION.md) | **`korg-redact@v1`** — redaction at persistence: secrets are removed *before* an event is hashed, plus the verifier lint (`korg-verify --require-redacted`) | +| [`IMPORT.md`](./IMPORT.md) + [`imports/`](./imports/) | **`korg-import@v1`** — allowlisted envelopes for sanitized Okto diagnostics and ningen-shikkaku lifecycle metadata (never secret values) | +| [`MIGRATION.md`](./MIGRATION.md) | what revision 1.1 changes for verifiers, producers, and vector consumers | | [`KORGCERT.md`](./KORGCERT.md) | **`korgcert@v1`** — the normative spec for the public, independently-verifiable *certificate*: a receipt superset that binds a re-derived (un-spoofable) human summary + an issuer Ed25519 seal. | -| [`vectors/`](./vectors/) + [`conformance.json`](./conformance.json) | the golden conformance vectors with **frozen tip hashes** — the cross-language oracle | +| [`vectors/`](./vectors/) + [`conformance.json`](./conformance.json) | the **only** copy of the golden conformance vectors: frozen tip hashes, exact causal-DAG errors, redaction lint results, and the shared redaction cases — the cross-language oracle | +| [`tools/generate_vectors.py`](./tools/generate_vectors.py) | deterministic generator for every vector + the manifest; `--check` (CI) fails on drift and asserts the frozen v1.0 tips | | [`conformance.py`](./conformance.py) | a dependency-free Python reference verifier (the executable oracle) | | [`js/`](./js/) | a dependency-free JavaScript verifier (`verify.mjs`) + its conformance harness, for Node and the browser | | [`web/`](./web/) | **two zero-install in-browser verifiers** (Web Crypto, send nothing): [`index.html`](./web/index.html) checks a raw `ledger.jsonl`; [`seal.html`](./web/seal.html) checks a **Certificate** — re-deriving the summary live so you can watch it match (or, when tampered, fail). Host on GitHub Pages and anyone verifies a shared session or seal in a tab. | @@ -34,7 +38,12 @@ An implementation in any language is **conformant** iff, given the vectors in 1. reports the intact chains as intact, **and** reproduces each frozen `tip_entry_hash` byte-for-byte (basic = `7418b910…`, HMAC = `1b371102…`); 2. flags each tampered vector at the named `seq`; -3. fails an HMAC vector verified without the key. +3. fails an HMAC vector verified without the key; +4. reproduces each vector's causal-DAG verdict, and for `dag: "broken"` the + exact `dag_errors` (flat **and** nested shapes, SPEC §9). + +Implementations that ship redaction also reproduce the shared cases in +`vectors/redaction/korg-redact-v1.json` and each vector's `redaction_errors`. Run a reference — each is one command, exit 0 = conformant: diff --git a/spec/korg-ledger-v1/REDACTION.md b/spec/korg-ledger-v1/REDACTION.md new file mode 100644 index 0000000..703d56e --- /dev/null +++ b/spec/korg-ledger-v1/REDACTION.md @@ -0,0 +1,166 @@ +# korg-redact@v1 — Redaction at Persistence + +**Status:** NORMATIVE (korg-ledger@v1 revision 1.1, [SPEC.md §10](./SPEC.md#10-redaction-at-persistence-revision-11)) · +**Cases:** [`vectors/redaction/korg-redact-v1.json`](./vectors/redaction/korg-redact-v1.json) · +**Implementations:** Python [`korg_ledger.redaction`](../../adapters/korg-ledger-py/src/korg_ledger/redaction.py), Rust [`korg_ledger::redact`](../../crates/korg-ledger/src/redact.rs) (feature `redact`) + +The key words MUST, MUST NOT, SHOULD and MAY are used as in RFC 2119. + +## 1. Why "at persistence" + +A korg ledger is a hash chain. Removing a secret from an event after it is hashed +changes the event's `entry_hash`, which every verifier reports as tampering — and +re-hashing the tail to hide that is exactly the forgery the chain exists to +catch. So there is one safe place to redact: **before the event exists**. + +## 2. Producer obligations + +**R1 — Order.** A producer MUST apply the policy to the record body, then +canonicalize, hash (§3), sign (§8.1), and write. No byte of an unredacted record +may reach disk, a log, a blob store, or a network sink owned by the ledger — no +write-then-rewrite. + +**R2 — Scope.** The policy applies to the whole record except the structural +fields `prev_hash`, `entry_hash`, `event_sig` (hex digests by construction). +For a nested JournalEvent that includes `metadata` and `event`; for a flat +record, every payload field. + +**R3 — Declaration.** A record produced under the policy MUST declare it: +`metadata.tags["korg.redaction"] = "korg-redact@v1"` (nested) or top-level +`"redaction": "korg-redact@v1"` (flat). The declaration is inside the preimage, +so it cannot be added or stripped later without breaking the chain. A record +without it makes no redaction claim. + +**R4 — Fail closed.** After redaction the producer MUST re-scan the record; if +it is not clean (see §5) the producer MUST NOT persist it and MUST surface an +error (`RedactionError` in Python). Under a correct implementation this is +unreachable because the policy is idempotent; it is the gate that makes the +guarantee unconditional. + +**R5 — No oracle.** A marker MUST NOT carry any function of the secret — no +hash, prefix, suffix, length, or entropy hint. A ledger is meant to be shared; +a hash of a low-entropy secret is a dictionary-attack oracle. + +**R6 — No retroactive redaction in place.** A persisted chained event MUST NOT +be edited to remove a secret. Remediation is re-chaining a redacted copy as a new +journal (see [`MIGRATION.md`](./MIGRATION.md#redacting-an-existing-journal)). + +**R7 — Opt-out is explicit and visible.** A producer MAY offer an explicit +opt-out (e.g. byte-exact fixture replay — `LedgerWriter(redaction=None)`); such +records carry no declaration (R3) and fail the lint if they hold a secret. + +## 3. The policy + +`redact(value) → (value', findings)` walks a JSON value. Object **keys are never +rewritten** (they are structure). Numbers, booleans, and null pass through. + +### 3.1 Sensitive keys + +Normalize an object key: insert `_` before an ASCII uppercase letter that follows +an ASCII lowercase letter or digit; ASCII-lowercase; map `-` and space to `_` +(`accessToken` → `access_token`, `X-Api-Key` → `x_api_key`, `APIKey` → +`apikey`). The key is **sensitive** if the normalized form is one of + +``` +authorization proxy_authorization cookie set_cookie api_key apikey password +passwd passphrase secret token private_key secret_key aws_secret_access_key +credential credentials +``` + +or ends with one of + +``` +_api_key _apikey _secret _secret_key _private_key _password _passwd _passphrase _token +``` + +(`prompt_tokens`, `max_tokens`, `token_count` are **not** sensitive.) The value +of a sensitive key is replaced — whole, whatever its type — by +`[REDACTED:sensitive_key]`, **except** `null`, booleans, `""`, and a string that +is already exactly a marker; those carry no secret and are kept. + +### 3.2 Value patterns + +Every other string is rewritten by these rules, **in this order**, each applied +to the output of the previous one. `B` is `(^|[^A-Za-z0-9_])` — an ASCII +boundary that is captured and re-emitted (no look-behind, no `\b`); in the +table `B sk-…` means `B` immediately followed by `sk-…` (the space is only for +readability). + +| Rule id | Pattern (regex) | Replacement | +|---|---|---| +| `private_key_block` | `-----BEGIN[A-Z ]*PRIVATE KEY-----(?:[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----\|[\s\S]*)` | marker | +| `url_credentials` | `([A-Za-z][A-Za-z0-9+.\-]*://)[^ \t\r\n/:@\[\]]+:[^ \t\r\n/@\[\]]+@` | `\1` marker `@` | +| `bearer_token` | `((?:Bearer\|bearer\|BEARER)[ \t]+)[A-Za-z0-9._~+/\-]{16,}=*` | `\1` marker | +| `provider_api_key` | `B sk-[A-Za-z0-9_\-]{20,}` | `\1` marker | +| `github_token` | `B (?:gh[pousr]_[A-Za-z0-9]{36,}\|github_pat_[A-Za-z0-9_]{22,})` | `\1` marker | +| `slack_token` | `B xox[abposr]-[A-Za-z0-9\-]{10,}` | `\1` marker | +| `aws_access_key_id` | `B (?:AKIA\|ASIA)[0-9A-Z]{16}` | `\1` marker | +| `google_api_key` | `B AIza[0-9A-Za-z_\-]{35}` | `\1` marker | +| `jwt` | `B eyJ[A-Za-z0-9_\-]{8,}\.eyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}` | `\1` marker | +| `env_assignment` | `B ([A-Z][A-Z0-9_]*(?:API_KEY\|APIKEY\|SECRET\|SECRET_KEY\|PASSWORD\|PASSWD\|TOKEN\|PRIVATE_KEY\|ACCESS_KEY))([ \t]*[=:][ \t]*)(?:"[^"\n]*"\|'[^'\n]*'\|[^ \t\r\n"',;&]+)` | `\1\2\3` marker | +| `json_secret_field` | `("(?:password\|passwd\|passphrase\|secret\|client_secret\|api_key\|apikey\|access_token\|refresh_token\|auth_token\|id_token\|token\|private_key\|authorization)"[ \t]*:[ \t]*)"(?:[^"\\\n]\|\\.)*"` | `\1"` marker `"` | + +The marker for rule `r` is the literal string `[REDACTED:r]`. + +**Fixed point.** The rule list is applied repeatedly until a full pass changes +nothing (at most 8 passes). One pass is not enough: a replacement can expose a +new match — `AKIA…AKIA…` back to back, where the first marker's `]` becomes the +boundary the second key needed. Iterating to the fixed point makes the policy +idempotent by construction; a string still changing after 8 passes is left +unclean and the producer's R4 gate refuses it. `userinfo` in `url_credentials` +excludes `[` and `]` (RFC 3986 gen-delims), so a marker is never mistaken for +credentials. + +**ASCII only, case-explicit.** The patterns deliberately avoid `\s`, `\w`, `\b`, +and case-insensitive flags: Python `re` and the Rust `regex` crate (and JS) +disagree on their Unicode semantics, and a disagreement would let one +implementation persist what another redacts. `ésk-…` is redacted (é is not an +ASCII word character) in every implementation; the shared cases pin this. + +### 3.3 Findings + +Each rule application that changed a string, and each sensitive-key +replacement, yields a finding `{path, rule}` where `path` is an RFC 6901 JSON +pointer (`""` for the root). Reported lists are sorted by `path` (code point) +then by the rule order above, with `sensitive_key` first — implementations +iterate object members in different orders, the sorted list is canonical. + +## 4. Properties (MUST hold; tested) + +- **Deterministic** — same input, same output, in every implementation (the + shared cases). +- **Idempotent** — `redact(redact(x)) == (redact(x), [])`; property-tested over + spliced fragments of every rule. +- **Non-expanding claims** — markers never match a rule in a way that changes + them, so an already-redacted ledger re-scans clean. + +## 5. The lint (verifiers) + +A record is **redaction-clean** iff `redact(body) == body`, where `body` is the +record minus `prev_hash`/`entry_hash`/`event_sig`. Journal lint errors, one per +finding, in journal order: + +``` +seq S: unredacted at +``` + +Run it with `korg-verify --require-redacted` (Rust; any failure makes +the verdict INVALID and sets `redaction_ok: false` in `--json`) or +`python -m korg_ledger lint `. The lint is a **floor, not a ceiling**: +a clean result means no *known* credential shape survived, not that the ledger +holds no sensitive data. Producers of structured sources SHOULD allowlist fields +instead (as the [import profiles](./IMPORT.md) do). + +## 6. Where it runs today + +| Persistence path | Redacts before hashing | Declares policy | +|---|---|---| +| `korg_ledger.LedgerWriter` (Python; Claude Code hook, `migrate`, imports) | yes (default) | `metadata.tags` | +| `POST /api/agent/tool-call` (korg-server; MCP agents) | yes — `args` and `result` | `metadata.tags` | +| korg-bridge `append*` (PyO3) | yes — `args` and `result` | `metadata.tags` | +| korg internal events (`CapabilityJournal::append`) | no — korg's own governance events carry no agent payload | — | +| `korg-runtime` run-once ledger (flat) | no — fixed internal pipeline fields | — | + +Known gaps (not covered by this policy): content-addressed blobs written by the +stream-json adapter to `.korg/blobs/` before the POST; object keys that are +themselves secrets; secrets with no recognizable shape under a non-sensitive key. diff --git a/spec/korg-ledger-v1/SPEC.md b/spec/korg-ledger-v1/SPEC.md index 0252bb6..85f228b 100644 --- a/spec/korg-ledger-v1/SPEC.md +++ b/spec/korg-ledger-v1/SPEC.md @@ -1,13 +1,28 @@ -# korg-ledger@v1 — Tamper-Evident Cognition Ledger - -**Status:** FROZEN · **Version:** `korg-ledger@v1` · **Reference:** [`src/ledger_spec.py`](../../src/ledger_spec.py) · **Conformance:** [`conformance.json`](./conformance.json) - -This document is the normative definition of korg's tamper-evident event ledger. -It is intentionally small and language-agnostic. Any implementation — the Rust -core (`korg-registry`), the Python reference (korgex), a JS verifier — is -**conformant** iff it reproduces the frozen [conformance vectors](./vectors/) -byte-for-byte. The reference module and this document MUST agree; the vectors -are the tie-breaker. +# korg-ledger@v1 — Tamper-Evident Agent Flight Recorder + +**Status:** AUTHORITATIVE · FROZEN hash rules · **Version:** `korg-ledger@v1`, revision **1.1** · **Reference oracle:** [`conformance.py`](./conformance.py) · **Conformance:** [`conformance.json`](./conformance.json) · **Vector generator:** [`tools/generate_vectors.py`](./tools/generate_vectors.py) + +This document is the normative definition of korg as an **agent flight +recorder**: the tamper-evident event ledger (§1–§8), how verifiers read both +journal shapes and their causal links (§9), redaction at persistence (§10, +[`REDACTION.md`](./REDACTION.md)), and import envelopes for evidence korg did not +observe itself (§11, [`IMPORT.md`](./IMPORT.md)). It is intentionally small and +language-agnostic. Any implementation — the Rust core (`korg-ledger`, +`korg-registry`, `korg-verify`), the Python producer (`korg_ledger`), the JS +verifier — is **conformant** iff it reproduces the [conformance +vectors](./vectors/) listed in [`conformance.json`](./conformance.json): frozen +tips, exact causal-DAG errors, and the shared redaction cases. + +### Authority + +**This directory is the canonical contract.** When any other document — the +repo-root `agent_event_spec.md` (the HTTP/MCP application profile), a README, a +code comment — or any implementation disagrees with this spec, this spec and its +vectors win and the other artifact is the bug. The vectors are the tie-breaker +between this prose and the reference oracle. There is exactly one copy of the +vectors and manifest (`vectors/`, `conformance.json`); every implementation reads +them from here, and `tools/generate_vectors.py --check` (CI) fails on any drift. +Migration notes for each revision: [`MIGRATION.md`](./MIGRATION.md). > The guarantee in one line: a journal is a hash-chain of events; any edit, > deletion, insertion, or reorder is detectable and localized to a `seq_id`. @@ -49,8 +64,9 @@ canonicalize({"z": [3, 2], "a": {"y": 1, "x": 2}}) == b'{"a":{"x":2,"y":1},"z" Non-ASCII is escaped to its `\uXXXX` form so the output is pure ASCII — e.g. a value of `"é"` serializes to the six-character escape `"é"`, never raw UTF-8. -(See the conformance test `test_canonicalize_is_sorted_compact_ascii` for the -exact byte-level assertion.) +(See the `canon non-ascii` / `canon astral surrogate-pair` assertions in +[`js/conformance.mjs`](./js/conformance.mjs) and the `nonbmp-intact.jsonl` +vector for the exact byte-level oracle.) ## 3. `entry_hash` @@ -87,9 +103,12 @@ Each error names the offending `seq_id`. A verifier given the wrong key (or no key for a keyed chain) MUST report tampering. `verify_dag(events) -> errors[]` additionally checks the causal structure: -`seq_id`s are unique, and every `triggered_by` references an existing, -**strictly earlier** `seq_id`. The strictly-earlier rule makes rewind-by- -truncation sound (cutting at seq N never orphans a survivor). +integer `seq_id`s are unique, and every event's causal parent (§9 — top-level +`triggered_by` on a flat record, `metadata.triggered_by` on a nested +JournalEvent) is a well-formed integer that references an existing, **strictly +earlier** `seq_id`. The strictly-earlier rule makes rewind-by-truncation sound +(cutting at seq N never orphans a survivor). DAG error strings are normative +(§9.4) so the three verifiers can be compared byte-for-byte. ## 6. Conformance @@ -102,11 +121,20 @@ truncation sound (cutting at seq N never orphans a survivor). `seq`. - the HMAC vector uses key `"korg-conformance-key"`; verifying it with no key MUST fail. +- every vector declares `dag`: `"intact"` vectors MUST `verify_dag == []`; + `"broken"` vectors MUST produce **exactly** the listed `dag_errors`, in order + (§9.4). +- vectors that declare `redaction` (`"clean"` / `"unclean"`) pin the + korg-redact@v1 lint: an implementation that ships the lint MUST produce + exactly the listed `redaction_errors`. The shared redaction cases + (`vectors/redaction/korg-redact-v1.json`) pin `redact()` itself. +- the manifest MUST list every `vectors/*.jsonl` file (no unlisted vectors). Run a conformance harness (exit 0 = conformant): `python3 spec/korg-ledger-v1/conformance.py` (Python), `node spec/korg-ledger-v1/js/conformance.mjs` (JavaScript), or -`cargo test -p korg-verify` (Rust). Regenerate vectors: -`python3 spec/korg-ledger-v1/_generate_vectors.py`. +`cargo test -p korg-verify` (Rust). Regenerate or drift-check vectors: +`python3 spec/korg-ledger-v1/tools/generate_vectors.py [--write]` — the v1.0 +vectors are frozen and the generator asserts it reproduces their tips. ## 7. v1 scope / non-goals @@ -171,3 +199,101 @@ A sidecar file (never in the preimage). One record per line: chain was published *before any third party fetched it*, not before a specific clock time. Anchors for seq_ids later removed by a rewind are stale and should be dropped. + +## 9. Journal shapes & causal parent (revision 1.1) + +### 9.1 Two shapes, one chain + +A journal is a sequence of JSON objects. Each object is one of two shapes; a +journal MAY mix them (verifiers decide per event): + +| Shape | Written by | Where the causal parent lives | +|---|---|---| +| **flat** — payload fields at the top level (`seq_id`, `tool_name`, `args`, …) | conformance vectors, `korg-runtime` run-once, receipts/Certificates minted from flat events, legacy adapters | top-level `triggered_by` | +| **nested** JournalEvent — `{schema_version, seq_id, metadata:{…}, event:{…}}` | `korg_ledger.LedgerWriter`, the Rust `CapabilityJournal` (korg-server, korg-bridge), the Claude Code hook | `metadata.triggered_by` | + +The hash chain (§2–§5) is shape-agnostic: the preimage is the whole top-level +object minus `HASH_FIELDS`, so `metadata` and `event` are hashed in full. +`seq_id`, `prev_hash`, `entry_hash`, and `event_sig` are always top-level. + +### 9.2 Ledger integers + +A **ledger integer** is a JSON number whose value is integral and within +±(2^53−1). JSON `true`/`false` are **not** integers (Python's `bool` subclasses +`int`; verifiers MUST NOT follow it). An integral number written with a fraction +or exponent (`2.0`, `2e0`) **is** the integer 2: a JS verifier cannot tell `2.0` +from `2` after parsing, so all verifiers accept it in order to agree. Producers +MUST still write plain integers (`2`). + +### 9.3 Resolving the causal parent + +`causal_parent(event)`: + +1. Collect the non-null values of top-level `triggered_by` and, when `metadata` + is an object, `metadata.triggered_by`. `null` and absence both mean "no + parent". +2. None collected → the event is a **root**. +3. Any collected value that is not a ledger integer → error + `triggered_by is not an integer` (a string `"3"`, `true`, `1.5`, `[1]`, an + integer beyond 2^53−1 — previously these were silently skipped, which made a + malformed link indistinguishable from a root). +4. Both collected and different → error + `conflicting triggered_by (top-level A vs metadata B)`. +5. Otherwise the parent is that integer. + +Then, as in §5, the parent MUST be the `seq_id` of an event in the journal and +MUST be strictly less than the event's own `seq_id`. + +### 9.4 Normative DAG errors + +`verify_dag` reports, in journal order (Rust, Python, JS, and the oracle emit +these byte-for-byte; `S` is the event's integer `seq_id` or `?`): + +``` +duplicate seq_id present (once, first) +seq S: triggered_by is not an integer +seq S: conflicting triggered_by (top-level A vs metadata B) +seq S: triggered_by T does not exist +seq S: triggered_by T is not strictly earlier +seq ?: triggered_by T cannot be ordered (seq_id is not an integer) +``` + +### 9.5 Write side + +Producers MUST validate the causal parent before appending: a ledger integer +naming an existing, strictly-earlier `seq_id`, or null. `LedgerWriter` raises +`CausalityError`; korg-bridge raises `CausalityError`; `POST +/api/agent/tool-call` answers **422** and appends nothing. + +## 10. Redaction at persistence (revision 1.1) + +A chained event cannot be edited after it is hashed — that is exactly what the +chain detects — so secrets MUST be removed **before** canonicalization, hashing, +signing, and the first byte written to disk. The normative policy +**`korg-redact@v1`** (sensitive-key and value-pattern rules, marker format, +idempotence, fail-closed gate, and the verifier lint) is specified in +[`REDACTION.md`](./REDACTION.md) and pinned by +`vectors/redaction/korg-redact-v1.json`. A record produced under the policy +declares it: `metadata.tags["korg.redaction"] = "korg-redact@v1"` (nested) or +top-level `"redaction": "korg-redact@v1"` (flat). Redaction never changes the +hash rules — a redacted journal is an ordinary v1 chain over redacted bytes. + +## 11. Import envelopes (revision 1.1) + +Evidence korg did not observe itself (Okto proxy diagnostics, ningen-shikkaku +lifecycle events) enters a ledger only through a **`korg-import@v1`** envelope +with a registered profile — `okto.diagnostics@v1` or +`ningen-shikkaku.lifecycle@v1`. Profiles are strict allowlists with no free-text +fields; secret-bearing source fields are named and rejected; validation is +atomic. Imported events carry `source_agent = "import:@"`, hang +off one `korg.import` batch event that records the envelope's SHA-256, and pass +through redaction like any other event. Normative text: +[`IMPORT.md`](./IMPORT.md); examples: [`imports/`](./imports/). + +## 12. Revision history + +| Revision | Change | Hash-compatible | Verdict-compatible | +|---|---|---|---| +| 1.0 | chain, HMAC, DAG over top-level `triggered_by` | — | — | +| 1.0 + §8 | `event_sig`, `anchors.jsonl` (additive, outside the preimage) | yes | yes | +| **1.1** | §9 causal parent read from both shapes + integer domain; §10 redaction at persistence; §11 import envelopes; one canonical vector set | **yes** — every v1.0 tip reproduces | **stricter**: a nested journal with a dangling/forward/malformed `metadata.triggered_by` that 1.0 verifiers silently passed now fails. See [`MIGRATION.md`](./MIGRATION.md). | diff --git a/spec/korg-ledger-v1/conformance.json b/spec/korg-ledger-v1/conformance.json index e19052c..df4ad8a 100644 --- a/spec/korg-ledger-v1/conformance.json +++ b/spec/korg-ledger-v1/conformance.json @@ -1,31 +1,109 @@ { "spec_version": "korg-ledger@v1", - "canonicalization": "JSON, keys sorted by code point, separators (',',':'), non-ASCII \\uXXXX-escaped; preimage = event minus entry_hash", + "revision": "1.1", + "canonicalization": "JSON, keys sorted by code point, separators (',',':'), non-ASCII \\uXXXX-escaped; preimage = event minus entry_hash and event_sig", "hmac": "HMAC-SHA256 over the same preimage when a key is present", + "causal_parent": "top-level triggered_by (flat) or metadata.triggered_by (nested); null/absent = root; both present must agree; value must be an integer (SPEC.md §9)", + "redaction_cases": "redaction/korg-redact-v1.json", "vectors": [ { "file": "basic-intact.jsonl", "key": null, "verify": "intact", + "dag": "intact", "tip_entry_hash": "7418b9105a664e21078fde881fbd8a5295c49bb384aa62a48d5b544292d910af" }, { "file": "hmac-intact.jsonl", "key": "korg-conformance-key", "verify": "intact", + "dag": "intact", "tip_entry_hash": "1b371102038082d3fac965fe21ba357129c61f57fcd93f1a0ffa5e6eaa454e78" }, + { + "file": "nonbmp-intact.jsonl", + "key": null, + "verify": "intact", + "dag": "intact", + "tip_entry_hash": "045d2d865e89c54340408b309c81d6b0f1367ebf0aa4504f1af66f7f1e4cb590" + }, { "file": "tampered-content.jsonl", "key": null, "verify": "tampered", - "error_contains": "seq 2" + "error_contains": "seq 2", + "dag": "intact" }, { "file": "tampered-deletion.jsonl", "key": null, "verify": "tampered", - "error_contains": "seq 3" + "error_contains": "seq 3", + "dag": "broken", + "dag_errors": [ + "seq 3: triggered_by 2 does not exist" + ] + }, + { + "file": "nested-intact.jsonl", + "key": null, + "verify": "intact", + "dag": "intact", + "tip_entry_hash": "d08d86d9a733e3edf24bf7f29658c975afdef0a84c144e4059e3bb96176a5b7b" + }, + { + "file": "nested-dag-broken.jsonl", + "key": null, + "verify": "intact", + "dag": "broken", + "dag_errors": [ + "seq 2: triggered_by 3 is not strictly earlier", + "seq 3: triggered_by 9 does not exist" + ], + "tip_entry_hash": "17bc6a0db4e13af116ef69b6a99bedeaf007b74a0a30f111774740d5a4481250" + }, + { + "file": "dag-malformed.jsonl", + "key": null, + "verify": "intact", + "dag": "broken", + "dag_errors": [ + "seq 2: triggered_by is not an integer", + "seq 3: triggered_by is not an integer", + "seq 4: conflicting triggered_by (top-level 2 vs metadata 1)", + "seq 5: triggered_by 5 is not strictly earlier", + "seq 6: triggered_by is not an integer" + ], + "tip_entry_hash": "750c832667893fb531a32fddcc95b3611c17637c30228fcd7a14043a4f513dab" + }, + { + "file": "mixed-shape-intact.jsonl", + "key": null, + "verify": "intact", + "dag": "intact", + "tip_entry_hash": "ea2176907fc8677190b954aef82b40366989613332d26227c698c4e72cd6ee12" + }, + { + "file": "redacted-intact.jsonl", + "key": null, + "verify": "intact", + "dag": "intact", + "redaction": "clean", + "tip_entry_hash": "c605cc63bbc9b56bc368919ab39b32c6830b5dc57238ed27e4696b61627f6623" + }, + { + "file": "unredacted-intact.jsonl", + "key": null, + "verify": "intact", + "dag": "intact", + "redaction": "unclean", + "redaction_errors": [ + "seq 2: unredacted provider_api_key at /event/args/command", + "seq 2: unredacted env_assignment at /event/args/command", + "seq 3: unredacted sensitive_key at /event/args/headers/Authorization", + "seq 3: unredacted url_credentials at /event/args/url" + ], + "tip_entry_hash": "2e7c3b3feff9784eabc9c08bf76fa89d95439ad0d17bf700afaf66b59fcee05f" } ] } diff --git a/spec/korg-ledger-v1/conformance.py b/spec/korg-ledger-v1/conformance.py index 6bb9805..c099800 100644 --- a/spec/korg-ledger-v1/conformance.py +++ b/spec/korg-ledger-v1/conformance.py @@ -51,14 +51,74 @@ def verify_chain(events: list, key: bytes | None = None) -> list: return errors +_SAFE_INT_MAX = 2**53 - 1 + + +def json_int(v): + # §9.2: an integral JSON number within ±(2^53-1); booleans are not integers. + if isinstance(v, bool): + return None + if isinstance(v, int): + return v if abs(v) <= _SAFE_INT_MAX else None + if isinstance(v, float) and v.is_integer() and abs(v) <= _SAFE_INT_MAX: + return int(v) + return None + + +def causal_parent(e: dict): + # §9.3: top-level triggered_by (flat) or metadata.triggered_by (nested); + # null/absent = root; when both are present they must agree. + raw = [e.get("triggered_by")] + md = e.get("metadata") + raw.append(md.get("triggered_by") if isinstance(md, dict) else None) + ints = [] + for v in (x for x in raw if x is not None): + n = json_int(v) + if n is None: + return None, "triggered_by is not an integer" + ints.append(n) + if len(ints) == 2 and ints[0] != ints[1]: + return None, f"conflicting triggered_by (top-level {ints[0]} vs metadata {ints[1]})" + return (ints[0] if ints else None), None + + +def verify_dag(events: list) -> list: + errors = [] + dicts = [e for e in events if isinstance(e, dict)] + seqs = [n for n in (json_int(e.get("seq_id")) for e in dicts) if n is not None] + if len(set(seqs)) != len(seqs): + errors.append("duplicate seq_id present") + for e in dicts: + sid = json_int(e.get("seq_id")) + label = "?" if sid is None else str(sid) + tb, err = causal_parent(e) + if err: + errors.append(f"seq {label}: {err}") + elif tb is None: + continue + elif tb not in set(seqs): + errors.append(f"seq {label}: triggered_by {tb} does not exist") + elif sid is None: + errors.append(f"seq ?: triggered_by {tb} cannot be ordered (seq_id is not an integer)") + elif tb >= sid: + errors.append(f"seq {label}: triggered_by {tb} is not strictly earlier") + return errors + + def _read(name): - with open(os.path.join(HERE, "vectors", name)) as f: + with open(os.path.join(HERE, "vectors", name), encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] def run() -> int: - manifest = json.load(open(os.path.join(HERE, "conformance.json"))) + with open(os.path.join(HERE, "conformance.json"), encoding="utf-8") as f: + manifest = json.load(f) assert manifest["spec_version"] == "korg-ledger@v1" + listed = {v["file"] for v in manifest["vectors"]} + on_disk = {n for n in os.listdir(os.path.join(HERE, "vectors")) if n.endswith(".jsonl")} + if listed != on_disk: + print(f" [FAIL] manifest/vectors out of sync: unlisted={sorted(on_disk - listed)} missing={sorted(listed - on_disk)}") + return 1 failures = 0 for v in manifest["vectors"]: events = _read(v["file"]) @@ -75,7 +135,12 @@ def run() -> int: ok, detail = False, "expected tampered, verified clean" elif not any(v["error_contains"] in e for e in errors): ok, detail = False, f"errors {errors} missing {v['error_contains']!r}" - print(f" [{'PASS' if ok else 'FAIL'}] {v['file']:<26} {v['verify']:<8} {detail}") + # §9 causal DAG — exact error strings are part of the oracle. + dag = verify_dag(events) + want_dag = v.get("dag_errors", []) if v.get("dag") == "broken" else [] + if ok and dag != want_dag: + ok, detail = False, f"dag errors {dag} != expected {want_dag}" + print(f" [{'PASS' if ok else 'FAIL'}] {v['file']:<26} {v['verify']:<8} dag={v.get('dag', '-'):<7} {detail}") failures += 0 if ok else 1 print(f"\nkorg-ledger@v1 conformance: {'PASS' if not failures else f'{failures} FAILURE(S)'}") return 0 if not failures else 1 diff --git a/spec/korg-ledger-v1/imports/ningen-shikkaku-lifecycle.example.json b/spec/korg-ledger-v1/imports/ningen-shikkaku-lifecycle.example.json new file mode 100644 index 0000000..def6455 --- /dev/null +++ b/spec/korg-ledger-v1/imports/ningen-shikkaku-lifecycle.example.json @@ -0,0 +1,15 @@ +{ + "schema": "korg-import@v1", + "profile": "ningen-shikkaku.lifecycle@v1", + "source": {"name": "ningen-shikkaku", "version": "0.1.0"}, + "records": [ + {"kind": "daemon_start", "mode": "armed", "pid": 4242, "grace_ms": 5000, "ping_timeout_ms": 3000, "seccomp": true}, + {"kind": "buffers", "allocated": 2, "locked": 2}, + {"kind": "pid_registered", "pid": 4300, "total": 1}, + {"kind": "status", "alive": true, "armed": true, "grace_seconds": 5, "registered_pids": 1}, + {"kind": "trigger", "reason": "heartbeat_dropped", "mode": "armed", "severity": "graceful"}, + {"kind": "panic", "reason": "grace_expired"}, + {"kind": "pid_killed", "pid": 4300, "outcome": "sent"}, + {"kind": "wiped", "buffers": 2} + ] +} diff --git a/spec/korg-ledger-v1/imports/ningen-shikkaku-lifecycle.rejected.json b/spec/korg-ledger-v1/imports/ningen-shikkaku-lifecycle.rejected.json new file mode 100644 index 0000000..3287376 --- /dev/null +++ b/spec/korg-ledger-v1/imports/ningen-shikkaku-lifecycle.rejected.json @@ -0,0 +1,9 @@ +{ + "schema": "korg-import@v1", + "profile": "ningen-shikkaku.lifecycle@v1", + "source": {"name": "ningen-shikkaku", "version": "0.1.0"}, + "records": [ + {"kind": "exit_condition_fired", "counter_mode": "calls", "calls": 1, "value": "synthetic-key-material"}, + {"kind": "trigger", "reason": "heartbeat connection dropped", "mode": "armed"} + ] +} diff --git a/spec/korg-ledger-v1/imports/okto-diagnostics.example.json b/spec/korg-ledger-v1/imports/okto-diagnostics.example.json new file mode 100644 index 0000000..e45b3c0 --- /dev/null +++ b/spec/korg-ledger-v1/imports/okto-diagnostics.example.json @@ -0,0 +1,21 @@ +{ + "schema": "korg-import@v1", + "profile": "okto.diagnostics@v1", + "source": {"name": "okto", "version": "0.1.0"}, + "exported_at_ms": 1781395260000, + "records": [ + {"kind": "session", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "client": "claude", + "at_ms": 1781395200000, "tls_established": true, "provider_vendor": "anthropic", + "provider_product": "messages-api", "turn_count": 1, "anomaly_count": 1, "mutation_count": 1}, + {"kind": "turn", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "turn_index": 0, + "status_code": 200, "duration_ms": 1840, "chunk_count": 42, "anomaly_count": 1, "mutations_applied": 1, + "token_usage": {"input_tokens": 1240, "output_tokens": 310}}, + {"kind": "anomaly", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "turn_index": 0, + "chunk_index": 17, "anomaly_type": "malformed_json_in_sse_frame", "repaired": true}, + {"kind": "mutation", "mutation_id": "9d8c7b6a-5e4f-4a3b-8c2d-1e0f9a8b7c6d", "scope": "stream_chunk", + "patch_type": "json_patch", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "client": "claude", + "confidence_bp": 9200, "reinforcement_count": 3, "at_ms": 1781395201840}, + {"kind": "replay", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "resolved": true, + "total_anomalies": 1, "total_mutations_applied": 1} + ] +} diff --git a/spec/korg-ledger-v1/imports/okto-diagnostics.rejected.json b/spec/korg-ledger-v1/imports/okto-diagnostics.rejected.json new file mode 100644 index 0000000..febb742 --- /dev/null +++ b/spec/korg-ledger-v1/imports/okto-diagnostics.rejected.json @@ -0,0 +1,12 @@ +{ + "schema": "korg-import@v1", + "profile": "okto.diagnostics@v1", + "source": {"name": "okto", "version": "0.1.0"}, + "records": [ + {"kind": "turn", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "turn_index": 0, + "request_headers": {"authorization": "Bearer sk-ant-api03-EXAMPLEEXAMPLEEXAMPLE"}}, + {"kind": "anomaly", "session_id": "3f2b9c1e-7a44-4d1b-9a0e-5c6d7e8f9a01", "turn_index": 0, + "chunk_index": 3, "anomaly_type": "malformed_json_in_sse_frame", + "description": "JSON parse failure in SSE data line: b'{\"api_key\": ...'"} + ] +} diff --git a/spec/korg-ledger-v1/js/conformance.mjs b/spec/korg-ledger-v1/js/conformance.mjs index 6e8eaa1..09fc32f 100644 --- a/spec/korg-ledger-v1/js/conformance.mjs +++ b/spec/korg-ledger-v1/js/conformance.mjs @@ -8,11 +8,12 @@ // // node conformance.mjs # exit 0 = this JS impl reproduces the vectors -import { readFileSync } from "node:fs"; +import { readFileSync, readdirSync } from "node:fs"; import { canonicalize, chainHash, verifyChain, + verifyDag, verifyEventSig, verifyAnchors, verifyKorgCert, @@ -83,7 +84,35 @@ async function run() { detail = `errors ${JSON.stringify(errors)} missing ${JSON.stringify(v.error_contains)}`; } } - console.log(` [${ok ? "PASS" : "FAIL"}] ${v.file.padEnd(26)} ${v.verify.padEnd(8)} ${detail}`); + // §9 causal DAG — exact error strings are part of the oracle. + const dag = verifyDag(events); + const wantDag = v.dag === "broken" ? v.dag_errors || [] : []; + if (ok && JSON.stringify(dag) !== JSON.stringify(wantDag)) { + ok = false; + detail = `dag errors ${JSON.stringify(dag)} != expected ${JSON.stringify(wantDag)}`; + } + console.log(` [${ok ? "PASS" : "FAIL"}] ${v.file.padEnd(26)} ${v.verify.padEnd(8)} dag=${(v.dag || "-").padEnd(7)} ${detail}`); + if (!ok) failures++; + } + { + const listed = new Set(manifest.vectors.map((v) => v.file)); + const onDisk = readdirSync(here("../vectors/")).filter((n) => n.endsWith(".jsonl")); + const ok = onDisk.length === listed.size && onDisk.every((n) => listed.has(n)); + console.log(` [${ok ? "PASS" : "FAIL"}] manifest lists every vector on disk`); + if (!ok) failures++; + } + // §9.2 integer domain for causal links: JS parses 2.0 as 2, so every impl + // accepts integral floats; booleans and fractions are never integers. + { + const mk = (tb) => [{ seq_id: 1 }, { seq_id: 2, triggered_by: tb }]; + const ok = + verifyDag(mk(1.0)).length === 0 && + verifyDag(mk(1.5))[0] === "seq 2: triggered_by is not an integer" && + verifyDag(mk(true))[0] === "seq 2: triggered_by is not an integer" && + verifyDag(mk(2 ** 60))[0] === "seq 2: triggered_by is not an integer" && + verifyDag([{ seq_id: 1 }, { seq_id: 2, metadata: { triggered_by: 3 } }])[0] === + "seq 2: triggered_by 3 does not exist"; + console.log(` [${ok ? "PASS" : "FAIL"}] causal-parent integer domain + nested lookup`); if (!ok) failures++; } diff --git a/spec/korg-ledger-v1/js/verify.mjs b/spec/korg-ledger-v1/js/verify.mjs index 2b59f88..c05f8ad 100644 --- a/spec/korg-ledger-v1/js/verify.mjs +++ b/spec/korg-ledger-v1/js/verify.mjs @@ -199,22 +199,69 @@ export async function verifyChain(events, keyBytes = null) { return errors; } -/** Check the causal DAG: unique seq_ids and strictly-earlier triggered_by links. */ +/** + * The integer a JSON value denotes, or null (korg-ledger@v1 §9.2): a number with + * an integral value within ±(2^53-1). Booleans are not integers; `2.0` parses to + * 2 here, which is why Python and Rust also accept integral floats. + */ +export function jsonInt(value) { + return typeof value === "number" && Number.isSafeInteger(value) ? value + 0 : null; +} + +function seqLabel(e) { + const n = jsonInt(e.seq_id); + return n === null ? "?" : String(n); +} + +/** + * Resolve an event's causal parent across the flat and nested journal shapes + * (§9.3): top-level `triggered_by` (flat) or `metadata.triggered_by` (nested + * JournalEvent); null and absence both mean root; when both are present they must + * agree. Returns `{ parent, error }`. + */ +export function causalParent(event) { + const raw = []; + if (event.triggered_by !== undefined && event.triggered_by !== null) raw.push(event.triggered_by); + const md = event.metadata; + if (md !== null && typeof md === "object" && !Array.isArray(md) && md.triggered_by !== undefined && md.triggered_by !== null) { + raw.push(md.triggered_by); + } + const parsed = []; + for (const v of raw) { + const n = jsonInt(v); + if (n === null) return { parent: null, error: "triggered_by is not an integer" }; + parsed.push(n); + } + if (parsed.length === 2 && parsed[0] !== parsed[1]) { + return { parent: null, error: `conflicting triggered_by (top-level ${parsed[0]} vs metadata ${parsed[1]})` }; + } + return { parent: parsed.length ? parsed[0] : null, error: null }; +} + +/** + * Check the causal DAG (§5 + §9): unique integer seq_ids, and every causal parent + * (flat or nested) is a well-formed integer referencing an existing, + * strictly-earlier seq_id. Error strings are byte-identical to Rust and Python. + */ export function verifyDag(events) { const errors = []; if (!Array.isArray(events)) return ["events is not a list"]; const dicts = events.filter((e) => e !== null && typeof e === "object" && !Array.isArray(e)); - const seqs = dicts.map((e) => e.seq_id).filter((s) => typeof s === "number"); + const seqs = dicts.map((e) => jsonInt(e.seq_id)).filter((s) => s !== null); const seqset = new Set(seqs); if (seqset.size !== seqs.length) errors.push("duplicate seq_id present"); for (const e of dicts) { - const tb = e.triggered_by; - if (typeof tb !== "number") continue; - const sid = e.seq_id; - if (!seqset.has(tb)) errors.push(`seq ${sid}: triggered_by ${tb} does not exist`); - else if (typeof sid === "number" && tb >= sid) { - errors.push(`seq ${sid}: triggered_by ${tb} is not strictly earlier`); + const label = seqLabel(e); + const { parent: tb, error } = causalParent(e); + if (error) { + errors.push(`seq ${label}: ${error}`); + continue; } + if (tb === null) continue; + const sid = jsonInt(e.seq_id); + if (!seqset.has(tb)) errors.push(`seq ${label}: triggered_by ${tb} does not exist`); + else if (sid === null) errors.push(`seq ?: triggered_by ${tb} cannot be ordered (seq_id is not an integer)`); + else if (tb >= sid) errors.push(`seq ${label}: triggered_by ${tb} is not strictly earlier`); } return errors; } diff --git a/spec/korg-ledger-v1/tools/diff_fuzz.py b/spec/korg-ledger-v1/tools/diff_fuzz.py index dbd18ae..8df2618 100644 --- a/spec/korg-ledger-v1/tools/diff_fuzz.py +++ b/spec/korg-ledger-v1/tools/diff_fuzz.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Differential fuzzer — prove the three independent korgcert@v1 verifiers AGREE. +"""Differential fuzzer — prove the three independent verifiers AGREE. + +Covers korgcert@v1 verdicts, the §9 causal-parent rules (flat vs nested, +integer domain, conflicts) and — Python vs Rust — the korg-redact@v1 lint. For each adversarially-mutated Certificate, run the Python, Rust, and JS verifiers and assert they return the same validity. Three independent codepaths written @@ -48,6 +51,14 @@ def _cli_valid(cmd: list[str]) -> object: return f"ERR({out.returncode}):{(out.stdout or out.stderr)[:60]!r}" +def _cli_field(cmd: list[str], field: str) -> object: + out = subprocess.run(cmd, capture_output=True, text=True) + try: + return json.loads(out.stdout).get(field) + except Exception: + return f"ERR({out.returncode}):{(out.stdout or out.stderr)[:60]!r}" + + def mutations(base: dict): yield "base", base @@ -187,6 +198,90 @@ def main() -> None: diverged += 1 print(f" [{'ok' if agree else 'DIVERGE':>7}] {'neg-seq bound anchor':16} py={py} rust={rust} js={js}") + # ── §9 causal parent: flat vs nested, integer domain, conflicts ────────── + # Each candidate is a correctly RE-CHAINED journal, so only the DAG verdict + # is under test. dag_ok must agree across Python, Rust and JS. + from korg_ledger import verify_dag + + def nested_ev(seq, tb): + return {"schema_version": "1.0", "seq_id": seq, + "metadata": {"event_id": f"e{seq}", "tags": {}, "triggered_by": tb}, + "event": {"event_type": "AgentToolCall", "tool_name": "t", "args": {}, "result": {}}} + + def rechain(evs): + prev = "0" * 64 + for e in evs: + e.pop("entry_hash", None) + e["prev_hash"] = prev + try: + e["entry_hash"] = chain_hash(e) + except ValueError: # out-of-domain value (2**60): chain verdict is not under test here + e["entry_hash"] = "f" * 64 + prev = e["entry_hash"] + return evs + + tb_values = [None, 1, 2, 1.0, 2.0, 1.5, True, False, "1", 2**60, -1, 0, 3, 99, [1], {"seq": 1}] + dag_cases = [] + for v in tb_values: + dag_cases.append((f"nested tb={v!r}", [nested_ev(1, None), nested_ev(2, 1), nested_ev(3, v)])) + flat3 = [{"seq_id": 1}, {"seq_id": 2, "triggered_by": 1}, {"seq_id": 3, "triggered_by": v}] + dag_cases.append((f"flat tb={v!r}", flat3)) + dag_cases.append(("conflict 1 vs 2", [nested_ev(1, None), nested_ev(2, 1), {**nested_ev(3, 2), "triggered_by": 1}])) + dag_cases.append(("agree 2 == 2.0", [nested_ev(1, None), nested_ev(2, 1), {**nested_ev(3, 2), "triggered_by": 2.0}])) + dag_cases.append(("bool seq_id dup", [{"seq_id": 1}, {"seq_id": True}, {"seq_id": 2, "triggered_by": 1}])) + dag_cases.append(("float seq_id", [{"seq_id": 1}, {"seq_id": 2.5, "triggered_by": 1}])) + for label, evs in dag_cases: + total += 1 + evs = rechain(copy.deepcopy(evs)) + with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) as f: + f.write("".join(json.dumps(e) + "\n" for e in evs)) + path = f.name + try: + py = verify_dag(json.loads(json.dumps(evs))) == [] + rust = _cli_field([str(RUST_BIN), path, "--json"], "dag_ok") + js = _cli_field(["node", str(VERIFY_MJS), path, "--json"], "dag_ok") + finally: + os.unlink(path) + agree = py == rust == js + if not agree: + diverged += 1 + print(f" [{'ok' if agree else 'DIVERGE':>7}] dag {label:28} py={py} rust={rust} js={js}") + + # ── korg-redact@v1: Python and Rust must flag exactly the same findings ── + import random + + from korg_ledger import verify_redaction + + frags = ["sk-", "abcdefghijklmnopqrstuvwxyz", "Bearer ", "API_KEY", "TOKEN", "=", ":", " ", "\n", "'", '"', + "https://", "u:p@", "@", "/", "[", "]", "AKIA", "IOSFODNN7EXAMPLE", "-----BEGIN RSA PRIVATE KEY-----", + "-----END RSA PRIVATE KEY-----", "eyJ", "abcdefgh.", '"password": ', "\\", "[REDACTED:jwt]", "é", + "中", "ghp_", "0123456789", "xoxb-", "AIza", "_", "-", "github_pat_", "\u001c", "\u00a0", "ſ"] + keys = ["cmd", "apiKey", "X-Api-Key", "max_tokens", "client-secret", "Authorization", "note", "OAuth2Token"] + rnd = random.Random(20261013) + evs = [] + for i in range(1, 401): + args = {rnd.choice(keys): "".join(rnd.choice(frags) for _ in range(rnd.randint(1, 14))) for _ in range(2)} + evs.append({"seq_id": i, "args": args}) + evs = rechain(evs) + total += 1 + with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False, encoding="utf-8") as f: + f.write("".join(json.dumps(e, ensure_ascii=False) + "\n" for e in evs)) + path = f.name + try: + py = verify_redaction(evs) + out = subprocess.run([str(RUST_BIN), path, "--require-redacted", "--json"], capture_output=True, text=True) + rust = [e for e in json.loads(out.stdout)["errors"] if ": unredacted " in e] + finally: + os.unlink(path) + agree = py == rust and len(py) > 0 + if not agree: + diverged += 1 + for a, b in zip(py, rust): + if a != b: + print(f" first difference: py={a!r} rust={b!r}") + break + print(f" [{'ok' if agree else 'DIVERGE':>7}] redaction lint, 400 random events: py={len(py)} rust={len(rust)} findings") + print(f"\ndifferential fuzz: {total} candidates · {diverged} divergence(s)") sys.exit(1 if diverged else 0) diff --git a/spec/korg-ledger-v1/tools/generate_vectors.py b/spec/korg-ledger-v1/tools/generate_vectors.py new file mode 100644 index 0000000..57992b5 --- /dev/null +++ b/spec/korg-ledger-v1/tools/generate_vectors.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Regenerate (or check) every korg-ledger@v1 conformance vector + the manifest. + +The single source for spec/korg-ledger-v1/vectors/*.jsonl and conformance.json. +Deterministic: no clocks, no random UUIDs. Hashes come from the stdlib oracle +(../conformance.py), never from a product implementation. + + python3 spec/korg-ledger-v1/tools/generate_vectors.py # --check (default): exit 1 on drift + python3 spec/korg-ledger-v1/tools/generate_vectors.py --write # rewrite vectors + manifest + +The v1.0 vectors are FROZEN: this script asserts it reproduces their original +tip hashes, so a regeneration can never silently move the oracle. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +SPEC = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SPEC)) + +from conformance import GENESIS, chain_hash # noqa: E402 + +VECTORS = SPEC / "vectors" +MANIFEST = SPEC / "conformance.json" +HMAC_KEY = "korg-conformance-key" + +# Tips frozen at korg-ledger@v1.0. Never edit these. +FROZEN_V1_0 = { + "basic-intact.jsonl": "7418b9105a664e21078fde881fbd8a5295c49bb384aa62a48d5b544292d910af", + "hmac-intact.jsonl": "1b371102038082d3fac965fe21ba357129c61f57fcd93f1a0ffa5e6eaa454e78", + "nonbmp-intact.jsonl": "045d2d865e89c54340408b309c81d6b0f1367ebf0aa4504f1af66f7f1e4cb590", +} + + +def chain(records, key=None): + """Assign prev_hash/entry_hash in order (records are mutated and returned).""" + prev = GENESIS + for r in records: + r["prev_hash"] = prev + r["entry_hash"] = chain_hash(r, key) + prev = r["entry_hash"] + return records + + +def flat(seq, tool, args, result, duration, triggered_by=None, agent="agent:conformance"): + r = { + "schema_version": "1.0", + "seq_id": seq, + "source_agent": agent, + "tool_name": tool, + "args": args, + "result": result, + "success": True, + "duration_ms": duration, + } + if triggered_by is not None: + r["triggered_by"] = triggered_by + return r + + +def nested(seq, tool, args, result, triggered_by, *, tags=None, physical=1781395200000): + """A JournalEvent exactly as LedgerWriter / the Rust registry serialize it, + with fixed ids and clock so the bytes are reproducible.""" + eid = f"00000000-0000-4000-8000-{seq:012d}" + causation = f"00000000-0000-4000-8000-{triggered_by:012d}" if isinstance(triggered_by, int) else None + return { + "schema_version": "1.0", + "seq_id": seq, + "metadata": { + "event_id": eid, + "correlation_id": "00000000-0000-0000-0000-000000000000", + "causation_id": causation, + "root_event_id": "00000000-0000-4000-8000-000000000001", + "actor_id": "korg:conformance", + "campaign_id": "00000000-0000-0000-0000-000000000000", + "emitted_at": {"physical": physical, "logical": seq - 1, "actor_id": 1}, + "branch_id": None, + "speculative": False, + "retry_count": 0, + "tier": "Telemetry", + "span_id": None, + "tags": dict(tags or {}), + "triggered_by": triggered_by, + }, + "event": { + "event_type": "AgentToolCall", + "source_agent": "agent:conformance", + "tool_name": tool, + "args": args, + "result": result, + "payload_refs": [], + "success": True, + "duration_ms": 0, + "timestamp": "2026-06-14T00:00:00Z", + }, + } + + +def legacy_basic(): + return [ + flat(1, "user_prompt", {"prompt": "add a function"}, {}, 0), + flat(2, "llm_inference", {"model": "m", "prompt_tokens": 10}, {"completion_tokens": 4}, 12, 1), + flat(3, "Write", {"path": "mathx.py"}, {"ok": True}, 3, 2), + ] + + +def build(): + """→ list of (filename, records, ensure_ascii, compact, manifest_entry).""" + out = [] + + # ── v1.0 (frozen) ──────────────────────────────────────────────────────── + basic = chain(legacy_basic()) + out.append(("basic-intact.jsonl", basic, True, False, + {"key": None, "verify": "intact", "dag": "intact"})) + + out.append(("hmac-intact.jsonl", chain(legacy_basic(), HMAC_KEY.encode()), True, False, + {"key": HMAC_KEY, "verify": "intact", "dag": "intact"})) + + nonbmp = chain([ + flat(1, "user_prompt", {"prompt": "make it \U0001F600 in 中文 \U00010000"}, {}, 0), + flat(2, "llm_inference", {"model": "m中", "prompt_tokens": 7}, + {"completion_tokens": 3, "note": "ok \U0001F4A9"}, 9, 1), + flat(3, "Write", {"path": "中文.py", "snippet": "# \U0001F600 \U00010000"}, {"ok": True}, 4, 2), + ]) + out.append(("nonbmp-intact.jsonl", nonbmp, False, False, + {"key": None, "verify": "intact", "dag": "intact"})) + + tampered = json.loads(json.dumps(basic)) + tampered[1]["args"]["model"] = "EVIL" + out.append(("tampered-content.jsonl", tampered, True, False, + {"key": None, "verify": "tampered", "error_contains": "seq 2", "dag": "intact"})) + + deletion = [json.loads(json.dumps(basic[0])), json.loads(json.dumps(basic[2]))] + out.append(("tampered-deletion.jsonl", deletion, True, False, + {"key": None, "verify": "tampered", "error_contains": "seq 3", + "dag": "broken", "dag_errors": ["seq 3: triggered_by 2 does not exist"]})) + + # ── v1.1: nested JournalEvent shape + causal-parent resolution (§9) ───── + out.append(("nested-intact.jsonl", chain([ + nested(1, "user_prompt", {"prompt": "fix the bug"}, {}, None), + nested(2, "llm_inference", {"model": "m"}, {"text": "reading the file"}, 1), + nested(3, "Edit", {"file_path": "math_utils.py"}, {"ok": True}, 2), + nested(4, "Bash", {"command": "pytest -q"}, {"exit_code": 0}, 2), + ]), True, True, {"key": None, "verify": "intact", "dag": "intact"})) + + out.append(("nested-dag-broken.jsonl", chain([ + nested(1, "user_prompt", {"prompt": "fix the bug"}, {}, None), + nested(2, "llm_inference", {"model": "m"}, {}, 3), + nested(3, "Edit", {"file_path": "math_utils.py"}, {"ok": True}, 9), + ]), True, True, { + "key": None, "verify": "intact", "dag": "broken", + "dag_errors": [ + "seq 2: triggered_by 3 is not strictly earlier", + "seq 3: triggered_by 9 does not exist", + ], + })) + + malformed = [ + flat(1, "user_prompt", {"prompt": "p"}, {}, 0), + flat(2, "llm_inference", {}, {}, 0, "1"), + flat(3, "Read", {}, {}, 0, True), + {**flat(4, "Edit", {}, {}, 0, 2), "metadata": {"triggered_by": 1}}, + flat(5, "Bash", {}, {}, 0, 5), + {**flat(6, "Bash", {}, {}, 0), "metadata": {"triggered_by": [1]}}, + ] + out.append(("dag-malformed.jsonl", chain(malformed), True, False, { + "key": None, "verify": "intact", "dag": "broken", + "dag_errors": [ + "seq 2: triggered_by is not an integer", + "seq 3: triggered_by is not an integer", + "seq 4: conflicting triggered_by (top-level 2 vs metadata 1)", + "seq 5: triggered_by 5 is not strictly earlier", + "seq 6: triggered_by is not an integer", + ], + })) + + mixed = [ + flat(1, "user_prompt", {"prompt": "p"}, {}, 0), + nested(2, "llm_inference", {"model": "m"}, {}, 1), + {**flat(3, "Edit", {"file_path": "a.py"}, {}, 0, 2), "metadata": {"triggered_by": 2}}, + {**flat(4, "Read", {"file_path": "a.py"}, {}, 0), "metadata": {"triggered_by": None}}, + ] + out.append(("mixed-shape-intact.jsonl", chain(mixed), True, True, + {"key": None, "verify": "intact", "dag": "intact"})) + + # ── v1.1: redaction at persistence (REDACTION.md) ──────────────────────── + tags = {"korg.redaction": "korg-redact@v1"} + out.append(("redacted-intact.jsonl", chain([ + nested(1, "user_prompt", {"prompt": "deploy with my key"}, {}, None, tags=tags), + nested(2, "Bash", {"command": "export OPENAI_API_KEY=[REDACTED:env_assignment] && ./deploy"}, + {"exit_code": 0}, 1, tags=tags), + nested(3, "WebFetch", {"url": "https://[REDACTED:url_credentials]@example.com/", + "headers": {"Authorization": "[REDACTED:sensitive_key]"}}, + {"status": 200}, 2, tags=tags), + ]), True, True, {"key": None, "verify": "intact", "dag": "intact", "redaction": "clean"})) + + out.append(("unredacted-intact.jsonl", chain([ + nested(1, "user_prompt", {"prompt": "deploy with my key"}, {}, None), + nested(2, "Bash", {"command": "export OPENAI_API_KEY=sk-conformanceAAAAAAAAAAAAAAAAAAAA && ./deploy"}, + {"exit_code": 0}, 1), + nested(3, "WebFetch", {"url": "https://bob:hunter2@example.com/", + "headers": {"Authorization": "Bearer conformance0000000000"}}, + {"status": 200}, 2), + ]), True, True, { + "key": None, "verify": "intact", "dag": "intact", "redaction": "unclean", + "redaction_errors": [ + "seq 2: unredacted provider_api_key at /event/args/command", + "seq 2: unredacted env_assignment at /event/args/command", + "seq 3: unredacted sensitive_key at /event/args/headers/Authorization", + "seq 3: unredacted url_credentials at /event/args/url", + ], + })) + return out + + +def render(records, ensure_ascii, compact): + seps = (",", ":") if compact else (", ", ": ") + return "".join(json.dumps(r, ensure_ascii=ensure_ascii, separators=seps) + "\n" for r in records) + + +def manifest_for(built): + vectors = [] + for name, records, _a, _c, entry in built: + v = {"file": name, **entry} + if entry["verify"] == "intact": + key = entry["key"].encode() if entry["key"] else None + v["tip_entry_hash"] = chain_hash(records[-1], key) + vectors.append(v) + return { + "spec_version": "korg-ledger@v1", + "revision": "1.1", + "canonicalization": "JSON, keys sorted by code point, separators (',',':'), non-ASCII \\uXXXX-escaped; preimage = event minus entry_hash and event_sig", + "hmac": "HMAC-SHA256 over the same preimage when a key is present", + "causal_parent": "top-level triggered_by (flat) or metadata.triggered_by (nested); null/absent = root; both present must agree; value must be an integer (SPEC.md §9)", + "redaction_cases": "redaction/korg-redact-v1.json", + "vectors": vectors, + } + + +def main(argv) -> int: + write = "--write" in argv + built = build() + for name, records, *_ in built: + if name in FROZEN_V1_0: + assert chain_hash(records[-1], HMAC_KEY.encode() if name.startswith("hmac") else None) == FROZEN_V1_0[name], ( + f"{name}: regenerated tip no longer matches the FROZEN v1.0 tip") + files = {name: render(records, a, c) for name, records, a, c, _ in built} + manifest = json.dumps(manifest_for(built), indent=2, ensure_ascii=False) + "\n" + + drift = [] + for name, text in files.items(): + path = VECTORS / name + if write: + path.write_text(text, encoding="utf-8") + elif not path.exists() or path.read_text(encoding="utf-8") != text: + drift.append(str(path.relative_to(SPEC))) + known = set(files) | {"redaction"} + stray = sorted(p.name for p in VECTORS.iterdir() if p.name not in known) + if write: + MANIFEST.write_text(manifest, encoding="utf-8") + elif MANIFEST.read_text(encoding="utf-8") != manifest: + drift.append("conformance.json") + + if stray: + print(f"vectors/ has files the generator does not own: {stray}") + if drift: + print("conformance vectors drifted from the generator:\n " + "\n ".join(drift)) + print("run: python3 spec/korg-ledger-v1/tools/generate_vectors.py --write") + if write: + print(f"wrote {len(files)} vectors + conformance.json") + return 1 if (drift or stray) and not write else 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/spec/korg-ledger-v1/vectors/dag-malformed.jsonl b/spec/korg-ledger-v1/vectors/dag-malformed.jsonl new file mode 100644 index 0000000..ff1ed82 --- /dev/null +++ b/spec/korg-ledger-v1/vectors/dag-malformed.jsonl @@ -0,0 +1,6 @@ +{"schema_version": "1.0", "seq_id": 1, "source_agent": "agent:conformance", "tool_name": "user_prompt", "args": {"prompt": "p"}, "result": {}, "success": true, "duration_ms": 0, "prev_hash": "0000000000000000000000000000000000000000000000000000000000000000", "entry_hash": "f1446e86d639a95a714a1c69db965d9ea8a60945db3c9d363beca285d8e9af48"} +{"schema_version": "1.0", "seq_id": 2, "source_agent": "agent:conformance", "tool_name": "llm_inference", "args": {}, "result": {}, "success": true, "duration_ms": 0, "triggered_by": "1", "prev_hash": "f1446e86d639a95a714a1c69db965d9ea8a60945db3c9d363beca285d8e9af48", "entry_hash": "9264886cdb6dcf049731ef8982660b177a715536627c49b6c3116cea7bf63100"} +{"schema_version": "1.0", "seq_id": 3, "source_agent": "agent:conformance", "tool_name": "Read", "args": {}, "result": {}, "success": true, "duration_ms": 0, "triggered_by": true, "prev_hash": "9264886cdb6dcf049731ef8982660b177a715536627c49b6c3116cea7bf63100", "entry_hash": "84f0981d6879c766150c79a9b3bc3c9ddf844f019d26b5f2fcd8db2b56058913"} +{"schema_version": "1.0", "seq_id": 4, "source_agent": "agent:conformance", "tool_name": "Edit", "args": {}, "result": {}, "success": true, "duration_ms": 0, "triggered_by": 2, "metadata": {"triggered_by": 1}, "prev_hash": "84f0981d6879c766150c79a9b3bc3c9ddf844f019d26b5f2fcd8db2b56058913", "entry_hash": "979543e1fbfd30b1c1c5cc5e90a42cde969e28ec108073c6d4e01f070cc3626a"} +{"schema_version": "1.0", "seq_id": 5, "source_agent": "agent:conformance", "tool_name": "Bash", "args": {}, "result": {}, "success": true, "duration_ms": 0, "triggered_by": 5, "prev_hash": "979543e1fbfd30b1c1c5cc5e90a42cde969e28ec108073c6d4e01f070cc3626a", "entry_hash": "4e222185d44631441707a3ec5e2314647d4e021c89fc9e793ce464012cccd867"} +{"schema_version": "1.0", "seq_id": 6, "source_agent": "agent:conformance", "tool_name": "Bash", "args": {}, "result": {}, "success": true, "duration_ms": 0, "metadata": {"triggered_by": [1]}, "prev_hash": "4e222185d44631441707a3ec5e2314647d4e021c89fc9e793ce464012cccd867", "entry_hash": "750c832667893fb531a32fddcc95b3611c17637c30228fcd7a14043a4f513dab"} diff --git a/spec/korg-ledger-v1/vectors/mixed-shape-intact.jsonl b/spec/korg-ledger-v1/vectors/mixed-shape-intact.jsonl new file mode 100644 index 0000000..ae79c07 --- /dev/null +++ b/spec/korg-ledger-v1/vectors/mixed-shape-intact.jsonl @@ -0,0 +1,4 @@ +{"schema_version":"1.0","seq_id":1,"source_agent":"agent:conformance","tool_name":"user_prompt","args":{"prompt":"p"},"result":{},"success":true,"duration_ms":0,"prev_hash":"0000000000000000000000000000000000000000000000000000000000000000","entry_hash":"f1446e86d639a95a714a1c69db965d9ea8a60945db3c9d363beca285d8e9af48"} +{"schema_version":"1.0","seq_id":2,"metadata":{"event_id":"00000000-0000-4000-8000-000000000002","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000001","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":1,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":1},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"llm_inference","args":{"model":"m"},"result":{},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"f1446e86d639a95a714a1c69db965d9ea8a60945db3c9d363beca285d8e9af48","entry_hash":"ac76060248ba385d6f5deba577954a100d71ec6134d967617f2b032723be7bcb"} +{"schema_version":"1.0","seq_id":3,"source_agent":"agent:conformance","tool_name":"Edit","args":{"file_path":"a.py"},"result":{},"success":true,"duration_ms":0,"triggered_by":2,"metadata":{"triggered_by":2},"prev_hash":"ac76060248ba385d6f5deba577954a100d71ec6134d967617f2b032723be7bcb","entry_hash":"6b0a029f114060d63c9addc6d205319f5cfac20abc398cf0d1bd678646a4c930"} +{"schema_version":"1.0","seq_id":4,"source_agent":"agent:conformance","tool_name":"Read","args":{"file_path":"a.py"},"result":{},"success":true,"duration_ms":0,"metadata":{"triggered_by":null},"prev_hash":"6b0a029f114060d63c9addc6d205319f5cfac20abc398cf0d1bd678646a4c930","entry_hash":"ea2176907fc8677190b954aef82b40366989613332d26227c698c4e72cd6ee12"} diff --git a/spec/korg-ledger-v1/vectors/nested-dag-broken.jsonl b/spec/korg-ledger-v1/vectors/nested-dag-broken.jsonl new file mode 100644 index 0000000..f26bc2e --- /dev/null +++ b/spec/korg-ledger-v1/vectors/nested-dag-broken.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"1.0","seq_id":1,"metadata":{"event_id":"00000000-0000-4000-8000-000000000001","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":null,"root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":0,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":null},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"user_prompt","args":{"prompt":"fix the bug"},"result":{},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"0000000000000000000000000000000000000000000000000000000000000000","entry_hash":"6f1cd265c45307b624776186294060d9492eea438398343d3804d550716eb6e4"} +{"schema_version":"1.0","seq_id":2,"metadata":{"event_id":"00000000-0000-4000-8000-000000000002","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000003","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":1,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":3},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"llm_inference","args":{"model":"m"},"result":{},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"6f1cd265c45307b624776186294060d9492eea438398343d3804d550716eb6e4","entry_hash":"073a4949fef2526cb49c4580c946162a709ef0615ae7faedf47ad7afeab40df5"} +{"schema_version":"1.0","seq_id":3,"metadata":{"event_id":"00000000-0000-4000-8000-000000000003","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000009","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":2,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":9},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"Edit","args":{"file_path":"math_utils.py"},"result":{"ok":true},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"073a4949fef2526cb49c4580c946162a709ef0615ae7faedf47ad7afeab40df5","entry_hash":"17bc6a0db4e13af116ef69b6a99bedeaf007b74a0a30f111774740d5a4481250"} diff --git a/spec/korg-ledger-v1/vectors/nested-intact.jsonl b/spec/korg-ledger-v1/vectors/nested-intact.jsonl new file mode 100644 index 0000000..6a0c0db --- /dev/null +++ b/spec/korg-ledger-v1/vectors/nested-intact.jsonl @@ -0,0 +1,4 @@ +{"schema_version":"1.0","seq_id":1,"metadata":{"event_id":"00000000-0000-4000-8000-000000000001","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":null,"root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":0,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":null},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"user_prompt","args":{"prompt":"fix the bug"},"result":{},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"0000000000000000000000000000000000000000000000000000000000000000","entry_hash":"6f1cd265c45307b624776186294060d9492eea438398343d3804d550716eb6e4"} +{"schema_version":"1.0","seq_id":2,"metadata":{"event_id":"00000000-0000-4000-8000-000000000002","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000001","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":1,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":1},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"llm_inference","args":{"model":"m"},"result":{"text":"reading the file"},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"6f1cd265c45307b624776186294060d9492eea438398343d3804d550716eb6e4","entry_hash":"69ee5cba59b4e0dfd75d2d2b4c83c6d99f71a011848c286597e36689d8b743df"} +{"schema_version":"1.0","seq_id":3,"metadata":{"event_id":"00000000-0000-4000-8000-000000000003","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000002","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":2,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":2},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"Edit","args":{"file_path":"math_utils.py"},"result":{"ok":true},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"69ee5cba59b4e0dfd75d2d2b4c83c6d99f71a011848c286597e36689d8b743df","entry_hash":"629f8a270b62100eb802dbd31377168210d3e6d5599e33dd9139ed2756a3d8b7"} +{"schema_version":"1.0","seq_id":4,"metadata":{"event_id":"00000000-0000-4000-8000-000000000004","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000002","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":3,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":2},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"Bash","args":{"command":"pytest -q"},"result":{"exit_code":0},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"629f8a270b62100eb802dbd31377168210d3e6d5599e33dd9139ed2756a3d8b7","entry_hash":"d08d86d9a733e3edf24bf7f29658c975afdef0a84c144e4059e3bb96176a5b7b"} diff --git a/crates/korg-registry/tests/conformance/nonbmp-intact.jsonl b/spec/korg-ledger-v1/vectors/nonbmp-intact.jsonl similarity index 100% rename from crates/korg-registry/tests/conformance/nonbmp-intact.jsonl rename to spec/korg-ledger-v1/vectors/nonbmp-intact.jsonl diff --git a/spec/korg-ledger-v1/vectors/redacted-intact.jsonl b/spec/korg-ledger-v1/vectors/redacted-intact.jsonl new file mode 100644 index 0000000..cf2b3fc --- /dev/null +++ b/spec/korg-ledger-v1/vectors/redacted-intact.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"1.0","seq_id":1,"metadata":{"event_id":"00000000-0000-4000-8000-000000000001","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":null,"root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":0,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{"korg.redaction":"korg-redact@v1"},"triggered_by":null},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"user_prompt","args":{"prompt":"deploy with my key"},"result":{},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"0000000000000000000000000000000000000000000000000000000000000000","entry_hash":"e991b48e69699da567c3ae7b33278597b70cb05b810b6331f0af876a53724287"} +{"schema_version":"1.0","seq_id":2,"metadata":{"event_id":"00000000-0000-4000-8000-000000000002","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000001","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":1,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{"korg.redaction":"korg-redact@v1"},"triggered_by":1},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"Bash","args":{"command":"export OPENAI_API_KEY=[REDACTED:env_assignment] && ./deploy"},"result":{"exit_code":0},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"e991b48e69699da567c3ae7b33278597b70cb05b810b6331f0af876a53724287","entry_hash":"5eda5c82886ff6dbf394ffa09e17d4a0e8dc23b49a4aabc64e2a47f812379be0"} +{"schema_version":"1.0","seq_id":3,"metadata":{"event_id":"00000000-0000-4000-8000-000000000003","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000002","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":2,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{"korg.redaction":"korg-redact@v1"},"triggered_by":2},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"WebFetch","args":{"url":"https://[REDACTED:url_credentials]@example.com/","headers":{"Authorization":"[REDACTED:sensitive_key]"}},"result":{"status":200},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"5eda5c82886ff6dbf394ffa09e17d4a0e8dc23b49a4aabc64e2a47f812379be0","entry_hash":"c605cc63bbc9b56bc368919ab39b32c6830b5dc57238ed27e4696b61627f6623"} diff --git a/spec/korg-ledger-v1/vectors/redaction/korg-redact-v1.json b/spec/korg-ledger-v1/vectors/redaction/korg-redact-v1.json new file mode 100644 index 0000000..bd0a55a --- /dev/null +++ b/spec/korg-ledger-v1/vectors/redaction/korg-redact-v1.json @@ -0,0 +1,361 @@ +{ + "policy": "korg-redact@v1", + "note": "Hand-authored expected outputs (not generated by any implementation). findings are in canonical order: JSON pointer by code point, then normative rule order. output null = input unchanged.", + "cases": [ + { + "name": "http_headers", + "input": { + "headers": { + "Authorization": "Bearer abcdefghijklmnopqrstuvwx", + "Accept": "application/json" + } + }, + "output": { + "headers": { + "Authorization": "[REDACTED:sensitive_key]", + "Accept": "application/json" + } + }, + "findings": [ + { + "path": "/headers/Authorization", + "rule": "sensitive_key" + } + ] + }, + { + "name": "key_normalization", + "input": { + "accessToken": "t0k", + "X-Api-Key": "k", + "clientSecret": "s", + "OPENAI_API_KEY": "v", + "Set-Cookie": "id=1", + "OAuth2Token": "o" + }, + "output": { + "accessToken": "[REDACTED:sensitive_key]", + "X-Api-Key": "[REDACTED:sensitive_key]", + "clientSecret": "[REDACTED:sensitive_key]", + "OPENAI_API_KEY": "[REDACTED:sensitive_key]", + "Set-Cookie": "[REDACTED:sensitive_key]", + "OAuth2Token": "[REDACTED:sensitive_key]" + }, + "findings": [ + { + "path": "/OAuth2Token", + "rule": "sensitive_key" + }, + { + "path": "/OPENAI_API_KEY", + "rule": "sensitive_key" + }, + { + "path": "/Set-Cookie", + "rule": "sensitive_key" + }, + { + "path": "/X-Api-Key", + "rule": "sensitive_key" + }, + { + "path": "/accessToken", + "rule": "sensitive_key" + }, + { + "path": "/clientSecret", + "rule": "sensitive_key" + } + ] + }, + { + "name": "non_secret_lookalikes_kept", + "input": { + "prompt_tokens": 10, + "max_tokens": 5, + "token_count": 3, + "tokens": [ + "a" + ], + "password_hash_algo": "argon2", + "secret": false, + "password": null, + "api_key": "" + }, + "output": null, + "findings": [] + }, + { + "name": "containers_and_numbers_under_sensitive_keys", + "input": { + "credentials": { + "user": "bob", + "password": "x" + }, + "token": 12345, + "auth": { + "refresh_token": [ + "a", + "b" + ] + } + }, + "output": { + "credentials": "[REDACTED:sensitive_key]", + "token": "[REDACTED:sensitive_key]", + "auth": { + "refresh_token": "[REDACTED:sensitive_key]" + } + }, + "findings": [ + { + "path": "/auth/refresh_token", + "rule": "sensitive_key" + }, + { + "path": "/credentials", + "rule": "sensitive_key" + }, + { + "path": "/token", + "rule": "sensitive_key" + } + ] + }, + { + "name": "existing_markers_are_stable", + "input": { + "password": "[REDACTED:sensitive_key]", + "note": "https://[REDACTED:url_credentials]@h/", + "cmd": "X_TOKEN=[REDACTED:env_assignment]" + }, + "output": null, + "findings": [] + }, + { + "name": "env_assignment", + "input": "export OPENAI_API_KEY=sk-abcdefghijklmnopqrstuvwxyz0123 && GITHUB_TOKEN='abc def' ./run", + "output": "export OPENAI_API_KEY=[REDACTED:env_assignment] && GITHUB_TOKEN=[REDACTED:env_assignment] ./run", + "findings": [ + { + "path": "", + "rule": "provider_api_key" + }, + { + "path": "", + "rule": "env_assignment" + } + ] + }, + { + "name": "env_assignment_yaml_and_dotenv", + "input": "DB_PASSWORD: hunter2\nAWS_SECRET_ACCESS_KEY=\"wJalr/K7MDENG\"\nDEBUG=1", + "output": "DB_PASSWORD: [REDACTED:env_assignment]\nAWS_SECRET_ACCESS_KEY=[REDACTED:env_assignment]\nDEBUG=1", + "findings": [ + { + "path": "", + "rule": "env_assignment" + } + ] + }, + { + "name": "env_lookalikes_kept", + "input": "MAX_TOKENS=4096 MY_TOKEN_FILE=/tmp/x token=abc Password=abc", + "output": null, + "findings": [] + }, + { + "name": "url_credentials", + "input": "git clone https://alice:s3cr3t@github.com/o/r.git; ssh git@github.com:o/r; https://bob@h/", + "output": "git clone https://[REDACTED:url_credentials]@github.com/o/r.git; ssh git@github.com:o/r; https://bob@h/", + "findings": [ + { + "path": "", + "rule": "url_credentials" + } + ] + }, + { + "name": "bearer_token", + "input": "curl -H 'Authorization: Bearer abc.def-ghi_jkl~mno+pqr/st==' https://x", + "output": "curl -H 'Authorization: Bearer [REDACTED:bearer_token]' https://x", + "findings": [ + { + "path": "", + "rule": "bearer_token" + } + ] + }, + { + "name": "bearer_prose_kept", + "input": "the bearer of bad news; BEARER short", + "output": null, + "findings": [] + }, + { + "name": "provider_api_keys", + "input": "a sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAA b sk-proj-0123456789abcdefghijKLMN c task-abcdefghijklmnopqrstuvwxyz", + "output": "a [REDACTED:provider_api_key] b [REDACTED:provider_api_key] c task-abcdefghijklmnopqrstuvwxyz", + "findings": [ + { + "path": "", + "rule": "provider_api_key" + } + ] + }, + { + "name": "adjacent_tokens", + "input": "sk-aaaaaaaaaaaaaaaaaaaaaaaa sk-bbbbbbbbbbbbbbbbbbbbbbbb", + "output": "[REDACTED:provider_api_key] [REDACTED:provider_api_key]", + "findings": [ + { + "path": "", + "rule": "provider_api_key" + } + ] + }, + { + "name": "ascii_boundary_before_non_ascii", + "input": "ésk-abcdefghijklmnopqrstuvwxyz 中AKIAIOSFODNN7EXAMPLE", + "output": "é[REDACTED:provider_api_key] 中[REDACTED:aws_access_key_id]", + "findings": [ + { + "path": "", + "rule": "provider_api_key" + }, + { + "path": "", + "rule": "aws_access_key_id" + } + ] + }, + { + "name": "vendor_tokens", + "input": [ + "ghp_A1b2C3d4E5A1b2C3d4E5A1b2C3d4E5F6g7H8", + "github_pat_11ABCDEFG011ABCDEFG0xy", + "xoxb-1234567890-abcdef", + "AKIAIOSFODNN7EXAMPLE", + "AIzaSyD-abcdefghijklmnopqrstuvwxyz01234", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U" + ], + "output": [ + "[REDACTED:github_token]", + "[REDACTED:github_token]", + "[REDACTED:slack_token]", + "[REDACTED:aws_access_key_id]", + "[REDACTED:google_api_key]", + "[REDACTED:jwt]" + ], + "findings": [ + { + "path": "/0", + "rule": "github_token" + }, + { + "path": "/1", + "rule": "github_token" + }, + { + "path": "/2", + "rule": "slack_token" + }, + { + "path": "/3", + "rule": "aws_access_key_id" + }, + { + "path": "/4", + "rule": "google_api_key" + }, + { + "path": "/5", + "rule": "jwt" + } + ] + }, + { + "name": "private_key_block", + "input": { + "file": "before\n-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA\nabc\n-----END RSA PRIVATE KEY-----\nafter", + "truncated": "x -----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXk" + }, + "output": { + "file": "before\n[REDACTED:private_key_block]\nafter", + "truncated": "x [REDACTED:private_key_block]" + }, + "findings": [ + { + "path": "/file", + "rule": "private_key_block" + }, + { + "path": "/truncated", + "rule": "private_key_block" + } + ] + }, + { + "name": "json_secret_field_in_text", + "input": "{\"password\": \"p\\\"w\", \"user\": \"u\", \"token\":\"t\", \"tokens\": \"n\"}", + "output": "{\"password\": \"[REDACTED:json_secret_field]\", \"user\": \"u\", \"token\":\"[REDACTED:json_secret_field]\", \"tokens\": \"n\"}", + "findings": [ + { + "path": "", + "rule": "json_secret_field" + } + ] + }, + { + "name": "pointer_escaping_and_arrays", + "input": { + "a/b~c": { + "api_key": "k" + }, + "list": [ + { + "password": "x" + }, + "Bearer abcdefghijklmnopqrstu" + ] + }, + "output": { + "a/b~c": { + "api_key": "[REDACTED:sensitive_key]" + }, + "list": [ + { + "password": "[REDACTED:sensitive_key]" + }, + "Bearer [REDACTED:bearer_token]" + ] + }, + "findings": [ + { + "path": "/a~1b~0c/api_key", + "rule": "sensitive_key" + }, + { + "path": "/list/0/password", + "rule": "sensitive_key" + }, + { + "path": "/list/1", + "rule": "bearer_token" + } + ] + }, + { + "name": "scalars_untouched", + "input": [ + 42, + null, + true, + -7, + "plain text" + ], + "output": null, + "findings": [] + } + ] +} diff --git a/spec/korg-ledger-v1/vectors/unredacted-intact.jsonl b/spec/korg-ledger-v1/vectors/unredacted-intact.jsonl new file mode 100644 index 0000000..302ec2c --- /dev/null +++ b/spec/korg-ledger-v1/vectors/unredacted-intact.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"1.0","seq_id":1,"metadata":{"event_id":"00000000-0000-4000-8000-000000000001","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":null,"root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":0,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":null},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"user_prompt","args":{"prompt":"deploy with my key"},"result":{},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"0000000000000000000000000000000000000000000000000000000000000000","entry_hash":"781db901e7abe5df5300c02a19d008f9fbb2a2225496137fe0882ddc29809df9"} +{"schema_version":"1.0","seq_id":2,"metadata":{"event_id":"00000000-0000-4000-8000-000000000002","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000001","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":1,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":1},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"Bash","args":{"command":"export OPENAI_API_KEY=sk-conformanceAAAAAAAAAAAAAAAAAAAA && ./deploy"},"result":{"exit_code":0},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"781db901e7abe5df5300c02a19d008f9fbb2a2225496137fe0882ddc29809df9","entry_hash":"c9bc915271657e3799e54d534e4812b08db19a472566be1d869404fd3b3d01cd"} +{"schema_version":"1.0","seq_id":3,"metadata":{"event_id":"00000000-0000-4000-8000-000000000003","correlation_id":"00000000-0000-0000-0000-000000000000","causation_id":"00000000-0000-4000-8000-000000000002","root_event_id":"00000000-0000-4000-8000-000000000001","actor_id":"korg:conformance","campaign_id":"00000000-0000-0000-0000-000000000000","emitted_at":{"physical":1781395200000,"logical":2,"actor_id":1},"branch_id":null,"speculative":false,"retry_count":0,"tier":"Telemetry","span_id":null,"tags":{},"triggered_by":2},"event":{"event_type":"AgentToolCall","source_agent":"agent:conformance","tool_name":"WebFetch","args":{"url":"https://bob:hunter2@example.com/","headers":{"Authorization":"Bearer conformance0000000000"}},"result":{"status":200},"payload_refs":[],"success":true,"duration_ms":0,"timestamp":"2026-06-14T00:00:00Z"},"prev_hash":"c9bc915271657e3799e54d534e4812b08db19a472566be1d869404fd3b3d01cd","entry_hash":"2e7c3b3feff9784eabc9c08bf76fa89d95439ad0d17bf700afaf66b59fcee05f"}