diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index b36efa758..753c06a0d 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 37 + max_docs: 38 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261031_120000_napetrov_session-prefilter.md b/dokimi/changelog.d/20261031_120000_napetrov_session-prefilter.md new file mode 100644 index 000000000..f94d6b043 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_session-prefilter.md @@ -0,0 +1,8 @@ + + +### Added + +- **`context prefilter`** — reads Claude Code session logs (`~/.claude/projects//*.jsonl`) and reports, per rule of an AGENTS.md or CLAUDE.md, how often the situation the rule describes came up, whether the agent ran the command or touched the path it names, and how often the user pushed back on it. The result is a ranked list of candidates for ablation, not a verdict: a rule that is rare or carries a protection tag (a negation, version gate, ordering or threshold) is listed but not ranked. Deterministic, no model call, no transcript text printed; `--json` gives the full report. Goose session stores (`sessions.db`) are read too. See [session-prefilter.md](docs/session-prefilter.md). diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 4133acaa6..2d3a97577 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -321,6 +321,9 @@ code. - [context-window.md](context-window.md) — read a run's context window from its Claude Code trace with `cli.py context trace`: start-up load, growth, compactions, tool-result weight. No LLM call. +- [session-prefilter.md](session-prefilter.md) — which AGENTS.md or CLAUDE.md rules + the Claude Code session logs triggered, acted on and were corrected on, with + `cli.py context prefilter`: candidates for ablation, not verdicts. No LLM call. - [run-consistency.md](run-consistency.md) — repeated runs of one task with `cli.py context consistency`: result stability (pass^k), path stability (tool mix, order, actions, strategies), process-contract conformance. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 5aafd5a88..5a2dc96fd 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -397,6 +397,11 @@ separate profiles: result (`pass@k`, `pass^k`), path (tool-mix JSD, edit distance of tool and action sequences, resource spread, strategy groups) and conformance to a YAML process contract; `python cli.py context consistency`. Guide: [run-consistency.md](run-consistency.md). +`dokimi/context/session_log.py` reads a Claude Code session log into prompt and +tool events; `dokimi/context/prefilter.py` matches a guidance file's rules +against them (trigger, acted, corrected) and ranks candidates for ablation. +Both are deterministic; `python cli.py context prefilter`. Guide: +[session-prefilter.md](session-prefilter.md). ## Shared building blocks diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 92f8caa80..76f6d1511 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -3392,6 +3392,142 @@ "subcommand_required": null, "subcommands": {} }, + "prefilter": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "guidance", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "sessions", + "kind": "argument", + "nargs": "+", + "position": 1, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.2, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--in-use-rate", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.context._unit_rate#9396aeedb2c0" + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-sessions", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 3, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--rare-max", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.context._non_negative_int#3a500817a521" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 10, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--window", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + } + ], + "path": "dokimi context prefilter", + "subcommand_required": null, + "subcommands": {} + }, "trace": { "allow_abbrev": true, "params": [ diff --git a/dokimi/docs/session-prefilter.md b/dokimi/docs/session-prefilter.md new file mode 100644 index 000000000..2cbe1d956 --- /dev/null +++ b/dokimi/docs/session-prefilter.md @@ -0,0 +1,141 @@ + + +# Guidance rules against session logs + +Measuring whether an AGENTS.md or CLAUDE.md helps means an ablation: run the +agent with and without a rule and compare outcomes. That is expensive, so +`context prefilter` picks which rules are worth one. It reads the coding +agent's own session logs and asks, per rule: did the situation the rule +describes come up, did the agent act on it, and did the user correct the agent +on it. The output is a ranked list of **candidates for ablation**. It is not a +verdict, and it never says a rule should be cut. + +Trigger rate is only a prefilter. A rule that rarely comes up can be the one +that prevents the costly failure, so a rare or protected rule is listed and +never ranked as waste. + +```bash +python cli.py context prefilter AGENTS.md "$HOME/.claude/projects//*.jsonl" +python cli.py context prefilter AGENTS.md "$HOME/.claude/projects//*.jsonl" --json +``` + +The guidance file is split into rules with the span segmentation the novelty +layer uses (`dokimi/novelty/spans.py`): each sentence, list item or table row +that has a keyword is a claim. Headings and code blocks are not claims. No +model is called; every match is lexical. + +## What is measured per rule + +- **Trigger.** An *episode* where the situation came up: a prompt the user + typed, or a tool call's inputs (command, paths, search patterns, description), + share at least half of the rule's keywords (stemmed, stop words dropped, + rounded up). A rule with no prose keywords (only backticked commands or paths, or a + protection tag) triggers on a prompt that names one of them. + Repeats within the window after a trigger are the same episode. +- **Acted.** Within `--window` events either side of the trigger, a tool call + ran a command or touched a path the rule names in backticks (`` `make ci-local` ``, + `` `changelog.d/` ``). A command counts when it runs the named command at the start of + a pipeline stage or compound command (after `uv run`, `sudo`, environment assignments, or + `then`, `do`, `(`), so an `echo`, a `grep` argument or a `#` comment that names it does not. + The key is read the same way as the command, so a wrapper in a key (`` `python -m pytest` ``) + matches the same wrapped command, and a quoted argument with a space is one word on both sides. A path counts when a tool call + names it in a path input or shell command, excluding shell comments, heredoc bodies, + echo/printf arguments (their redirection targets still count) and quoted sentences: a quoted + argument with a space in it is text, so `git commit -m "update docs/guide.md"` does not act on + the file, while `cat "docs/guide.md"` does. A quoted path that itself contains a space is read as a + sentence and does not count. A `>` inside quotes is text, not a redirection. A `<` reads its file, and a + shell function's definition does not count as acting: its body is ignored (up to the brace that closes it, + quotes and nested groups counted), and calls of the function are not followed. + A capitalised bare file name in backticks (`` `Makefile` ``) matches an edit of that file, or a + shell command that names it as an argument (`cat Makefile`); an `echo` of it does not. A lowercase bare word + (`` `pytest` ``) is a command, so an argument naming it (`grep pytest`) does not count. A path with a + space in its name (`` `docs/my guide.md` ``) is a path, not a command. A Windows path + matches with backslashes too. + A rule that names neither is not measurable here: its + `acted` is `null` and it carries the flag `acted_not_measurable`. +- **Corrected.** Within `--window` events after the trigger, the user's next + prompt (the first prompt after it, and only that one) pushes back (`no`, `don't`, `instead`, `forgot`, `revert`, ...) and + shares a keyword with the rule or names its command or path. Two or more + episodes with corrections on half or more of them carry `user_overrides`: + check the wording before anything else. + +Not counted: the agent's own text, tool results (what a file or command +printed), subagent briefs (a `user` record with `isSidechain`), and records +Claude Code injects (`isMeta`, such as CLAUDE.md echoed into the context). A +memory echo of a rule would otherwise trigger that rule on every session. + +## Statuses + +| Status | Meaning | Ranked | +| --- | --- | --- | +| `candidate_for_ablation` | Triggered in fewer than `--in-use-rate` of the sessions, or never. Ranked fewest triggered sessions first, then lowest acted share. | yes | +| `in_use` | Triggered in at least `--in-use-rate` of the sessions. | no | +| `rare_not_ranked` | Triggered in 1 through `--rare-max` sessions. Few sessions are too little evidence to call a rule unused. | no | +| `kept_protected` | The rule carries a protection tag: a negation, a version gate, an ordering constraint or a threshold (the same tags the novelty layer uses). Its failure case is rare by design. | no | +| `insufficient_corpus` | Fewer than `--min-sessions` sessions were read. | no | + +Precedence: `kept_protected`, then `insufficient_corpus`, then a rule with no trigger +(a candidate), then `rare_not_ranked`, then `in_use`; the rest are candidates. `--rare-max` +takes 0 or more and `--in-use-rate` a number from 0 to 1. A file named twice (a repeat, an +overlapping glob, a symlink to it) is read once. + +The table prints the same counts; the JSON has every claim with its status, +keys (commands, paths, keywords), counts, rates and flags. Defaults: window 10 +events, at least 5 sessions, rare at most 3 sessions, in use at 20% of sessions. + +The JSON is printed, not stored: dokimi writes no file in this format, so it +has no schema under `dokimi/schemas/`. `format_version` marks its shape. A +change that stores it must first add a versioned schema there. + +## Session logs read + +This slice reads **Claude Code** session logs: JSON lines under +`~/.claude/projects//`, with `user` and `assistant` records and +`tool_use` blocks. A quoted glob expands. A file that cannot be read, or holds +no prompt and no tool call, is reported and the rest are still read; the exit +code is 1 when any file failed. A line that is valid JSON but not an object, or a +record whose message or content has the wrong shape, is reported by line number and +skipped; summaries and tool results are recognized and not reported. + +**Goose** sessions are read from its SQLite store, `sessions.db` under the Goose +data directory (`~/.local/share/goose/sessions/` on Linux). Each session in the +store is one session here. The layout comes from the block/goose source +(session storage and message serialization), not from a sample store: a +message's `content_json` is a list of blocks, a `text` block of a user message +is a prompt, and a `toolRequest` block carries the tool's arguments. A message whose +content is not a list of blocks is reported by message id and skipped. A message +whose `metadata_json` marks it not user-visible is not a prompt. The store is +opened read-only. Pass the `.db` file in place of, or next to, JSONL logs. The +reader is `dokimi/context/goose_log.py`. + +## Privacy and offline use + +The command reads session logs and the guidance file on the local disk and +prints counts and the guidance's own rule text. It prints no transcript text, +calls no model and opens no network connection. Warnings name a file, not its +content. The tests use synthetic fixtures only. + +## Limits + +- Keyword matching is lexical. A rule phrased with words the agent never uses + for its situation has a low trigger rate for that reason alone, which is why + a rule's text is read before any ablation, not after. +- "Acted" needs a named command or path. Most prose rules have none; for them + the report says `null`, not zero. +- A quoted argument with no space is read as a path, so a commit message of one + word that names a file (`git commit -m "docs/guide.md"`) counts as acting on it. +- Corrections are read from the next prompts only, so a correction given much + later is missed. + +An LLM matcher could judge each episode's relevance and whether the agent +complied. It is not implemented here. A plausible shape: one call per session +over the rules whose keywords matched, which for a corpus of 200 sessions is +roughly 200 calls with a rule list and a short excerpt each. That is an +estimate, not a measurement. + +Related: [context-window.md](context-window.md) (what one run's window held), +[run-consistency.md](run-consistency.md) (whether repeated runs agree), +[skill-novelty.md](skill-novelty.md) (the rule-level novelty of a skill). diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 2c1947ed9..2c994a097 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -158,6 +158,7 @@ class Tombstone: Leaf(("tools", "preference"), PAID), Leaf(("context", "trace"), FREE), Leaf(("context", "consistency"), FREE), + Leaf(("context", "prefilter"), FREE), Leaf(("runs", "list"), FREE), Leaf(("runs", "show"), FREE), Leaf(("backends",), FREE), diff --git a/dokimi/dokimi/commands/context.py b/dokimi/dokimi/commands/context.py index fba02c217..58d2b94cf 100644 --- a/dokimi/dokimi/commands/context.py +++ b/dokimi/dokimi/commands/context.py @@ -6,7 +6,9 @@ from __future__ import annotations import argparse +import glob import json +import math import os import sys from pathlib import Path @@ -148,6 +150,76 @@ def cmd_context_consistency(args: argparse.Namespace) -> None: sys.exit(1) +def cmd_context_prefilter(args: argparse.Namespace) -> None: + """Which guidance claims the sessions triggered, acted on, or were corrected on: candidates for ablation.""" + from dokimi.context.prefilter import build_report, render_table + + guidance = Path(args.guidance) + try: + text = guidance.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + sys.exit(f"✗ cannot read {guidance}: {exc}") + + sessions, failed = _read_sessions(args.sessions) + if not sessions: + sys.exit("✗ no session could be read") + report = build_report(guidance.name, text, sessions, window=args.window, min_sessions=args.min_sessions, + rare_max=args.rare_max, in_use_rate=args.in_use_rate) + if not report["claims"]: + print(f"warning: no rules found in {guidance}", file=sys.stderr) + print(json.dumps(report, indent=2) if args.json else render_table(report)) + if failed: + sys.exit(1) + + +def _read_sessions(names: list[str]) -> tuple[list, bool]: + """Every readable session of ``names``: JSONL logs or Goose stores; a quoted glob expands. + + A file is read once however often it is named (a repeat, a glob that overlaps, a symlink to it). + """ + from dokimi.context.goose_log import is_goose_store, read_goose_store + from dokimi.context.session_log import SessionError, read_session + + loaded, failed, seen = [], False, set() + for name in names: + expanded = os.path.expanduser(name) # a literal file is checked after ~ is expanded, before any glob + matches = [expanded] if os.path.exists(expanded) else sorted(glob.glob(expanded, recursive=True)) + for path in matches or [expanded]: + real = os.path.realpath(path) + if real in seen: + print(f"warning: {path} given twice; counted once", file=sys.stderr) + continue + seen.add(real) + unreadable: list = [] # Goose sessions of unreadable messages only: not counted, still reported + try: + source = Path(path) + sessions = (read_goose_store(source, unreadable) if is_goose_store(source) + else [read_session(source)]) + except SessionError as exc: # report it, and still read the sessions that do + print(f"✗ {exc}", file=sys.stderr) + failed = True + sessions = [] + for session in [*sessions, *unreadable]: + for problem in session.errors: + print(f"warning: {session.name}: {problem}", file=sys.stderr) + loaded.extend(sessions) + return loaded, failed + + +def _non_negative_int(value: str) -> int: + number = int(value) + if number < 0: + raise argparse.ArgumentTypeError(f"must be 0 or more, not {value}") + return number + + +def _unit_rate(value: str) -> float: + rate = float(value) + if not math.isfinite(rate) or not 0 <= rate <= 1: + raise argparse.ArgumentTypeError(f"must be a number from 0 to 1, not {value}") + return rate + + def _unique_traces(label: str, names: list[str]) -> list[str]: """Each trace once: a repeated ``--arm`` or overlapping globs must not count one run twice.""" seen: dict[str, str] = {} @@ -237,3 +309,24 @@ def register(sub, positive_int) -> None: "count as edits; call: one step per tool call") cons_p.add_argument("--json", action="store_true", help="Print the profiles as JSON") cons_p.set_defaults(func=cmd_context_consistency) + + pre_p = context_sub.add_parser( + "prefilter", + help="Which rules of an AGENTS.md or CLAUDE.md the sessions triggered, acted on and were " + "corrected on, read from Claude Code session logs: a ranked list of candidates for " + "ablation, not verdicts. No LLM call.", + ) + pre_p.add_argument("guidance", help="The guidance file whose rules are measured (AGENTS.md, CLAUDE.md)") + pre_p.add_argument("sessions", nargs="+", + help="Claude Code session logs (`~/.claude/projects//*.jsonl`); a quoted glob expands") + pre_p.add_argument("--window", type=positive_int, default=10, + help="Events either side of a trigger that count as acting on it, and after it for a " + "correction (default 10)") + pre_p.add_argument("--min-sessions", type=positive_int, default=5, + help="Fewer sessions than this: no candidates are ranked (default 5)") + pre_p.add_argument("--rare-max", type=_non_negative_int, default=3, + help="A rule triggered in at most this many sessions is rare: listed, not ranked (default 3)") + pre_p.add_argument("--in-use-rate", type=_unit_rate, default=0.2, + help="A rule triggered in at least this share of sessions is in use, not a candidate (default 0.2)") + pre_p.add_argument("--json", action="store_true", help="Print the full report as JSON") + pre_p.set_defaults(func=cmd_context_prefilter) diff --git a/dokimi/dokimi/context/goose_log.py b/dokimi/dokimi/context/goose_log.py new file mode 100644 index 000000000..b5f493ef8 --- /dev/null +++ b/dokimi/dokimi/context/goose_log.py @@ -0,0 +1,99 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A Goose session store (``~/.local/share/goose/sessions/sessions.db``) -> one session per row. + +Goose keeps messages in SQLite: ``messages.session_id`` names the session and ``content_json`` +is a list of blocks. A ``text`` block of a ``user`` message is a prompt, unless the message's +``metadata_json`` marks it not user-visible (a note injected for the agent). A ``toolRequest`` +block of an ``assistant`` message is a tool call, its arguments under ``toolCall.value``. +Results (``toolResponse``) are not events. The layout comes from the block/goose source (session +storage and message serialization). The store is opened read-only; nothing is written. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any + +from dokimi.context.session_log import Event, Session, SessionError, add_error, prompt_text, tool_event + +SQLITE_MAGIC = b"SQLite format 3\x00" + + +def is_goose_store(path: Path) -> bool: + try: + with path.open("rb") as handle: + return handle.read(len(SQLITE_MAGIC)) == SQLITE_MAGIC + except OSError: + return False + + +def read_goose_store(path: Path, dropped: list[Session] | None = None) -> list[Session]: + """The sessions of a Goose store that hold a prompt or a tool call, oldest first. + + A session with no prompt or call is not counted; given ``dropped``, it is appended there so its + diagnostics can still be reported. + """ + try: + conn = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) + except sqlite3.Error as exc: + raise SessionError(f"cannot read {path}: {exc}") from exc + try: + ids = [sid for (sid,) in conn.execute("SELECT id FROM sessions ORDER BY created_at, id")] + sessions = [_session_of(conn, sid, f"{path}#{sid}") for sid in ids] + except sqlite3.Error as exc: + raise SessionError(f"{path}: not a Goose sessions store: {exc}") from exc + finally: + conn.close() + kept = [s for s in sessions if s.events] # a session of unreadable messages only is no session + if dropped is not None: + dropped.extend(s for s in sessions if not s.events and s.errors) + if not kept: + raise SessionError(f"{path}: no prompt and no tool call in any session") + return kept + + +def _session_of(conn: sqlite3.Connection, session_id: str, name: str) -> Session: + session = Session(name=name) + rows = conn.execute( + "SELECT id, role, content_json, metadata_json FROM messages WHERE session_id = ? " + "ORDER BY created_timestamp, id", (session_id,)).fetchall() + for row_id, role, content_json, metadata_json in rows: + try: + content = json.loads(content_json or "[]") + except (ValueError, RecursionError): # not JSON, or nested too deeply + add_error(session, f"message {row_id}: content is not JSON") + continue + if not isinstance(content, list): + add_error(session, f"message {row_id}: content is not a list of blocks") + continue + session.events.extend(_events_of(role, content, _user_visible(metadata_json))) + return session + + +def _user_visible(metadata_json: str | None) -> bool: + """A message without the mark counts as the user's; only ``userVisible: false`` excludes it.""" + try: + meta = json.loads(metadata_json) if metadata_json else {} + except (ValueError, RecursionError): + return True + return not (isinstance(meta, dict) and meta.get("userVisible") is False) + + +def _events_of(role: str, content: list[Any], user_visible: bool) -> list[Event]: + events = [] + if role == "user" and user_visible: # the text blocks of one message are one prompt + texts = [b.get("text") for b in content if isinstance(b, dict) and b.get("type") == "text"] + words = prompt_text(" ".join(t for t in texts if isinstance(t, str))) + if words: + events.append(Event("prompt", words)) + for block in content: + if isinstance(block, dict) and role == "assistant" and block.get("type") == "toolRequest": + call = block.get("toolCall") + value = call.get("value") if isinstance(call, dict) else None + if isinstance(value, dict): + events.append(tool_event(value.get("arguments"))) + return events diff --git a/dokimi/dokimi/context/prefilter.py b/dokimi/dokimi/context/prefilter.py new file mode 100644 index 000000000..53ae893c6 --- /dev/null +++ b/dokimi/dokimi/context/prefilter.py @@ -0,0 +1,542 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Which guidance claims the sessions ever brought up, acted on, or were corrected on: a prefilter for ablation. + +A guidance file (AGENTS.md, CLAUDE.md) is split into claims with the novelty layer's span +segmentation. Each claim gets three deterministic measures, read against session logs: + +* **trigger**: an episode where the situation the claim describes came up. A prompt or a tool + call shares half the claim's stemmed keywords (rounded up) with it. +* **acted**: within ``window`` events either side of the trigger, a tool call ran a command or + touched a path the claim names in backticks. A claim naming neither is not measurable. +* **corrected**: within ``window`` events after the trigger, the user's next prompt pushes back + on the same subject. + +The output is a ranked list of **candidates for ablation**, never a verdict: an ablation run +decides. A claim with a protection tag (negation, version gate, ordering, threshold) or with +few triggers is listed but not ranked, since a rare safety rule looks like waste on a count. +""" + +from __future__ import annotations + +import functools +import re +from dataclasses import dataclass +from typing import Any + +from dokimi.context import shell +from dokimi.context.session_log import Event, Session +from dokimi.novelty.spans import segment + +CANDIDATE = "candidate_for_ablation" +IN_USE = "in_use" +RARE = "rare_not_ranked" +KEPT = "kept_protected" +INSUFFICIENT = "insufficient_corpus" +STATUSES = (CANDIDATE, IN_USE, RARE, KEPT, INSUFFICIENT) +NOTE = ("Candidates for ablation, not verdicts: a candidate is a rule the sessions rarely brought up. " + "Run an ablation before changing any rule. Protected and rare rules are listed, not ranked.") + +_RULE_KINDS = frozenset({"sentence", "list_item", "table_row"}) +_BACKTICK = re.compile(r"`([^`\n]+)`") +_WORD = re.compile(r"[a-z][a-z0-9_-]{2,}") +_PATH = re.compile(r"[/\\]|\.[A-Za-z0-9]{1,6}$") +# a quoted argument is text, not a command; a backslash escapes the next character, also inside double quotes +_QUOTED = re.compile(r"\\[\s\S]|'[^']*'|\"(?:\\[\s\S]|[^\"\\])*\"") +# an unquoted backslash-newline is a line continuation: the line goes on, it is not a new command +_CONTINUATION = re.compile(r"\\\n|\\[\s\S]|'[^']*'|\"(?:\\[\s\S]|[^\"\\])*\"") +# a heredoc header is ``<()'\"]+))") +# a ``#`` starts a comment at the start of a word: the start of a line, after blank space or a control operator +_COMMENT = re.compile(r"(?m)(?:^|(?<=[\s;&|()]))#.*$") +# ``$((1<<2))`` is arithmetic, not a heredoc +_ARITHMETIC = re.compile(r"\$\(\([\s\S]*?\)\)") +# a comment, or a quoted piece of shell text (group 1), scanned left to right: an apostrophe in a comment opens no quote +# and a ``#`` in quotes is text +_COMMENT_OR_QUOTED = re.compile(r"(?m)(\\[\s\S]|'[^']*'|\"(?:\\[\s\S]|[^\"\\])*\")|(?:^|(?<=[\s;&|()]))#.*$") +# a redirection operator at the start of a word (``>``, ``>>``, ``2>``, ``<``); a bare one takes the next word +_REDIRECT_OP = re.compile(r"\d*(?:<<<|<<-?|>>|>&|<&|>\||[<>])") +_SEGMENT = re.compile(r"&&|\|\||(?])[|&]|[;()\n]") # ``>|``, ``>&``, ``<&`` are operators +_CUE = re.compile(r"\b(no|not|don't|do not|stop|instead|wrong|forgot|should have|revert|undo|why did you)\b", re.I) +_STOP = frozenset(""" +able about above after again against all also always and any are because been before being below +between both but by can could did does doing done down during each either else ensure even ever every +file files code for from further had has have having her here hers him his how however into its just +keep last like make may might more most must myself need neither new nor not now off once only onto +other ought our out over own run same see set should since some such than that the their theirs them +then there these they this those through to too under until upon use used using very via was way we +were what when where whether which while who whom why will with within without would yet you your +""".split()) + + +@dataclass(frozen=True) +class Claim: + """One checkable rule of a guidance file, with the words and names its triggers are matched on.""" + + id: str + text: str + heading: str + keywords: frozenset[str] + commands: tuple[str, ...] + paths: tuple[str, ...] + protection: tuple[str, ...] + + @property + def has_keys(self) -> bool: + return bool(self.commands or self.paths) + + +@dataclass +class Tally: + """How one claim's triggers, actions and corrections fell across a corpus.""" + + sessions_triggered: int = 0 + episodes: int = 0 + acted: int = 0 + corrected: int = 0 + + +def _stem(word: str) -> str: + for suffix in ("ing", "ed", "es", "s"): + if word.endswith(suffix) and len(word) - len(suffix) >= 4: + return word[: -len(suffix)] + return word + + +@functools.lru_cache(maxsize=8192) # every claim is matched against the same event texts +def _stems(text: str) -> frozenset[str]: + return frozenset(_stem(w) for w in _WORD.findall(text.lower()) if w not in _STOP) + + +def _is_path(key: str) -> bool: + """A bare file name or a path, or a path with a space in its name (``docs/my guide.md``): its first word is + a directory part and its last ends in an extension. A command line that names a file is a command.""" + if " " not in key: + return bool(_PATH.search(key)) + words = shell.tokens(key) + return (len(words) >= 2 and bool(re.search(r"[/\\]", words[0])) and not any(w.startswith("-") for w in words) + and bool(re.search(r"\.[A-Za-z0-9]{1,6}$", words[-1]))) + + +def claims_from_guidance(text: str) -> list[Claim]: + """The rules of one guidance file: each sentence, list item or table row with a keyword.""" + claims = [] + for span in segment(text): + if span.kind not in _RULE_KINDS: + continue + keywords = _stems(_BACKTICK.sub(" ", span.text)) + keys = [k.strip() for k in _BACKTICK.findall(span.text) if k.strip()] + if not (keywords or keys or span.protection): # a rule with nothing to match on is no claim + continue + claims.append(Claim( + id=span.id, text=span.text.strip(), heading=" > ".join(span.heading_path), keywords=keywords, + commands=tuple(k for k in keys if not _is_path(k)), + paths=tuple(k for k in keys if _is_path(k)), protection=tuple(span.protection), + )) + return claims + + +def _names(key: str, text: str) -> bool: + """Whether ``key`` occurs in ``text`` as a whole command, not inside a longer word.""" + words = " ".join(key.split()) + return bool(words) and re.search(rf"(? str: + """A quoted or escaped piece of shell text as the word it stands for: its quotes go, its literal characters stay.""" + text = match.group(0) + if text[0] == "\\": + return text[1:] + if text[0] == "'": + return text[1:-1] + return re.sub(r'\\([\\"$`])', r"\1", text[1:-1]) + + +def _dequote(text: str) -> str: + return _QUOTED.sub(_dequote_match, text) + + +# A separator, a ``#`` or a redirection sign inside quotes is text, not shell syntax: it is swapped for a +# private-use character while the command is split and cut, and swapped back where the text is used. +_SPECIAL = ";|&()\n#<>" +_TO_MASK = str.maketrans(_SPECIAL, "".join(chr(0xE000 + i) for i in range(len(_SPECIAL)))) +_FROM_MASK = str.maketrans("".join(chr(0xE000 + i) for i in range(len(_SPECIAL))), _SPECIAL) +_HIDDEN = " " # stands in for a quoted sentence in a path text: none of its words count as a path + + +def _strip_comments(text: str, *, hide_spaced: bool = False) -> str: + """The text with its comments removed and its quoted pieces masked. With ``hide_spaced`` a quoted argument with + a space in it is a sentence, not a path, and is replaced by a placeholder (one word is kept).""" + def piece(match: re.Match[str]) -> str: + quoted = match.group(1) + if quoted is None: + return "" + if hide_spaced and quoted[0] in "'\"" and any(c.isspace() for c in quoted[1:-1]): + return _HIDDEN + return quoted.translate(_TO_MASK) + return _COMMENT_OR_QUOTED.sub(piece, text) + + +def _join_line(match: re.Match[str]) -> str: + return "" if match.group(0) == "\\\n" else match.group(0) + + +def _blank(match: re.Match[str]) -> str: + return "_" * len(match.group(0)) # same length: positions in the line are kept + + +def _heredoc_words(line: str) -> list[tuple[str, bool]]: + """The heredocs a line opens, as (WORD, whether ``<<-`` strips tabs). A ``<<`` inside quotes, in a comment or + in arithmetic is text and opens none.""" + masked = _COMMENT.sub(_blank, _ARITHMETIC.sub(_blank, _QUOTED.sub(_blank, line))) + opened_words = [] + for opened in _HEREDOC_OPEN.finditer(masked): + word = _HEREDOC_WORD.match(line, opened.end()) + if word: + name = word.group(2) or word.group(3) or word.group(4) + opened_words.append((name, word.group(1) == "-")) + return opened_words + + +def _without_heredoc_bodies(command: str) -> str: + """The command with each heredoc body dropped: the lines after its header up to a line that is its WORD + alone. Body lines are read as data, so their quotes do not pair with the rest of the command.""" + out: list[str] = [] + pending: list[tuple[str, bool]] = [] # the WORDs of the bodies still to skip, in order + for line in command.split("\n"): + if pending: + name, strip_tabs = pending[0] + # a plain ``<<`` closes only on a WORD alone; ``<<-`` also on one indented with tabs + if (line.lstrip("\t") if strip_tabs else line) == name: + pending.pop(0) + continue + out.append(line) + pending.extend(_heredoc_words(line)) + return "\n".join(out) + + +def _function_end(text: str, open_brace: int) -> int: + """The index just past the ``}`` that closes the ``{`` at ``open_brace``, or -1 when the body never closes. Quoted + text and comments are skipped, so a brace in them does not count, and nested groups are counted.""" + depth, i = 0, open_brace + while i < len(text): + skipped = _QUOTED.match(text, i) or _COMMENT.match(text, i) + if skipped: + i = skipped.end() + continue + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return -1 + + +# quoted text and comments are matched first, so a function head inside them is text; a head is ``name() {`` +_SCAN = re.compile(r"(?m)(?P\\[\s\S]|'[^']*'|\"(?:\\[\s\S]|[^\"\\])*\"|(?:^|(?<=[\s;&|()]))#.*$)" + r"|(?P\b[A-Za-z_][\w-]*\s*\(\s*\)\s*\{)") + + +def _without_functions(text: str) -> str: + """The text with each shell function definition replaced by a space: its body is defined, not run. A head in + quotes or a comment is text. A body that never closes is not a definition, so the text after it still runs.""" + out, pos = [], 0 + for match in _SCAN.finditer(text): + if match.group("head") is None or match.start() < pos: + continue + end = _function_end(text, match.end() - 1) + if end < 0: + continue + out += [text[pos:match.start()], " "] + pos = end + return "".join(out) + text[pos:] + + +def _executable(command: str) -> str: + """The lines a shell would run: heredoc bodies, line continuations and function definitions left out.""" + text = _CONTINUATION.sub(_join_line, _without_heredoc_bodies(command or "")) + return _without_functions(text) + + +def _masked_segments(command: str, *, hide_spaced: bool = False) -> list[str]: + """The command's segments with their quotes removed, still masked. A heredoc body is dropped; with + ``hide_spaced`` a quoted argument with a space in it is replaced by a placeholder.""" + masked = _strip_comments(_executable(command), hide_spaced=hide_spaced) + return [_dequote(seg) for seg in _SEGMENT.split(masked)] + + +def _words(seg: str) -> str: + return " ".join(seg.translate(_FROM_MASK).split()) + + +def _command_segments(command: str) -> list[str]: + """The simple commands of a shell line, quotes kept: split at separators and pipes outside quotes, with + heredoc bodies, line continuations and comments left out. Read here, not by ``shell.segments``, because that + walker cuts a heredoc at a ``<<`` inside quotes and would hide the command after it.""" + masked = _strip_comments(_executable(command)) + return [seg.translate(_FROM_MASK) for seg in _SEGMENT.split(masked)] + + +def _words_of(segment: str) -> list[str]: + """A simple command as shell words, past its compound-command syntax (``then``, ``do``, ``(``).""" + return shell.tokens(shell.clean(segment)) + + +def _without_redirections(words: list[str]) -> list[str]: + """The words of a command with its redirections left out: ``>log``, ``2>&1``, ``2 >log`` and ``> log`` are not + arguments. The tokenizer splits a descriptor (``2``) from its operator, so a number before one is dropped too.""" + out: list[str] = [] + i = 0 + while i < len(words): + word, i = words[i], i + 1 + if word.isdigit() and i < len(words) and _REDIRECT_OP.match(words[i]): + continue + op = _REDIRECT_OP.match(word) + if op: + if op.end() == len(word): # a bare operator: the next word is its target + i += 1 + continue + out.append(word) + return out + + +def _key_words(key: str) -> list[str]: + """A guidance key as shell words, with its wrappers stripped the way a command's are (``python -m pytest`` + is ``pytest``), unless that leaves nothing.""" + words = shell.tokens(key) + return shell.strip_wrappers(words) or words + + +def _invokes(key: str, command: str) -> bool: + """Whether a shell command runs ``key``: as the program of one of its pipeline stages, after the wrappers + (``sudo -n``, ``env -u X``, ``timeout 600``, ``uv run``, ``A=1``), so an ``echo``, a ``grep`` argument or a + ``#`` comment that names it does not count. The key and the command are read the same way, so a quoted + argument with a space is one word on both sides. The key ends where its last word ends: ``make ci-local`` + does not run ``make ci-local.debug``, while a redirection after it (``make ci-local>log``) still runs it.""" + words = _key_words(key) + if not words: + return False + return any(shell.strip_wrappers(_without_redirections(_words_of(seg)))[:len(words)] == words + for seg in _command_segments(command)) + + +def _prints(text: str) -> bool: + """Whether a segment is an ``echo`` or ``printf``: its arguments are printed, not read or written. Compound + syntax (``then``, ``do``, ``else``) is not the program.""" + words = shell.strip_wrappers(_without_redirections(_words_of(text))) + return bool(words) and words[0] in ("echo", "printf") + + +# a redirection target is written (``>``, ``>>``, ``>|``, ``>&``) or read (``<``); ``<<`` is a heredoc +_REDIRECT = re.compile(r"(?()]+)|>(?:>|[|&])?\s*([^\s;&|<>()]+)") + + +@functools.lru_cache(maxsize=8192) # every claim is matched against the same tool calls +def _shell_paths(command: str) -> str: + """The text of a shell command that can name a file it reads or writes: its redirection targets and the + segments that are not printing commands. Comments, heredoc bodies, printed text and quoted sentences are + left out; a redirection sign inside quotes is not a redirection.""" + parts: list[str] = [] + for seg in _masked_segments(command, hide_spaced=True): + parts.extend((m.group(1) or m.group(2)).translate(_FROM_MASK) for m in _REDIRECT.finditer(seg)) + text = _words(seg) + if not _prints(text): + parts.append(text) + return " ".join(parts) + + +def _triggers(claim: Claim, event: Event) -> bool: + """Half the claim's keywords (rounded up) in the event: one word of a two-word rule is enough. + + A rule with no prose keywords triggers on a prompt that names one. + """ + if not claim.keywords: + return event.kind == "prompt" and _named(claim, event.text) + return len(claim.keywords & _stems(event.text)) >= (len(claim.keywords) + 1) // 2 + + +def _slashes(text: str) -> str: + return text.replace("\\", "/") # a Windows path is the same path with backslashes + + +def _at_path(key: str, text: str) -> bool: + """Whether ``key`` occurs in ``text`` from a path-segment start, so ``a.md`` does not match ``data.md``.""" + name = _slashes(key).removeprefix("./") + text = _slashes(text) + # a file name ends where the name ends (``a.md`` is not ``a.mdx`` or ``a.md.bak``); a directory + # (``docs/``) is followed by its entries, so it has no end boundary + end = "" if name.endswith("/") else r"(?![\w-]|\.\w)" + return bool(name) and re.search(rf"(? bool: + return any(_at_path(k, t) for k in claim.paths for t in texts) + + +def _named(claim: Claim, text: str) -> bool: + """Whether a prompt's text names one of the claim's commands or paths.""" + return any(_names(k, text) for k in claim.commands) or _path_named(claim, (text,)) + + +def _acts(claim: Claim, event: Event) -> bool: + if event.kind != "tool": + return False + shell_text = _shell_paths(event.command) + return (any(_invokes(k, event.command) for k in claim.commands) + or any(_file_named(k, event.paths) for k in claim.commands if _is_bare_file(k)) + or _path_named(claim, (*event.paths, shell_text)) + or any(_at_path(k, shell_text) for k in claim.commands if _is_bare_file(k))) + + +# a capitalised bare name (Makefile, LICENSE) is a file; a lowercase one (pytest) is a command, and its argument +# in a grep or an echo names it without running it +_BARE_FILE = re.compile(r"[A-Z][A-Za-z0-9_-]*") + + +def _is_bare_file(key: str) -> bool: + return _BARE_FILE.fullmatch(key) is not None + + +def _file_named(key: str, paths: tuple[str, ...]) -> bool: + """Whether a touched file's name is ``key``: a bare file name such as ``Makefile`` is a command key, and an + edit of that file is the named file being acted on.""" + return any(_slashes(p).rsplit("/", 1)[-1] == key for p in paths) + + +def _corrects(claim: Claim, event: Event) -> bool: + if event.kind != "prompt" or not _CUE.search(event.text): + return False + return (bool(claim.keywords & _stems(event.text)) + or any(_names(k, event.text) for k in claim.commands) + or any(_at_path(k, event.text) for k in claim.paths)) + + +def _scan(claim: Claim, events: list[Event], window: int) -> tuple[int, int, int]: + """Episodes, acted and corrected counts of one session; one episode spans its trigger's window.""" + episodes = acted = corrected = 0 + i = 0 + while i < len(events): + if not _triggers(claim, events[i]): + i += 1 + continue + episodes += 1 + around = events[max(0, i - window): i + window + 1] + after = events[i + 1: i + window + 1] + acted += int(claim.has_keys and any(_acts(claim, e) for e in around)) + next_prompt = next((e for e in after if e.kind == "prompt"), None) # the user's next word, only + corrected += int(next_prompt is not None and _corrects(claim, next_prompt)) + i += window + 1 + return episodes, acted, corrected + + +def tally(claim: Claim, sessions: list[Session], window: int) -> Tally: + out = Tally() + for session in sessions: + episodes, acted, corrected = _scan(claim, session.events, window) + out.episodes += episodes + out.acted += acted + out.corrected += corrected + out.sessions_triggered += int(episodes > 0) + return out + + +def status_of(claim: Claim, tally_: Tally, sessions_total: int, *, min_sessions: int, + rare_max: int, in_use_rate: float) -> str: + if claim.protection: + return KEPT + if sessions_total < min_sessions: + return INSUFFICIENT + if tally_.sessions_triggered == 0: + return CANDIDATE + if tally_.sessions_triggered <= rare_max: + return RARE + if tally_.sessions_triggered / sessions_total >= in_use_rate: + return IN_USE + return CANDIDATE + + +def _flags(claim: Claim, tally_: Tally) -> list[str]: + flags = [] + if not claim.has_keys: + flags.append("acted_not_measurable") + if tally_.episodes >= 2 and tally_.corrected * 2 >= tally_.episodes: + flags.append("user_overrides: the user pushed back on half or more of the triggers; check the wording") + return flags + + +def _ratio(part: int, whole: int) -> float | None: + return round(part / whole, 3) if whole else None + + +def _row(claim: Claim, t: Tally, status: str, sessions_total: int) -> dict[str, Any]: + return { + "id": claim.id, + "rank": None, + "status": status, + "text": claim.text if len(claim.text) <= 200 else claim.text[:197] + "...", + "heading": claim.heading, + "protection": list(claim.protection), + "keys": {"commands": list(claim.commands), "paths": list(claim.paths), "keywords": sorted(claim.keywords)}, + "sessions_triggered": t.sessions_triggered, + "trigger_rate": _ratio(t.sessions_triggered, sessions_total), + "episodes": t.episodes, + "acted": t.acted if claim.has_keys else None, + "acted_rate": _ratio(t.acted, t.episodes) if claim.has_keys else None, + "corrected": t.corrected, + "correction_rate": _ratio(t.corrected, t.episodes), + "flags": _flags(claim, t), + } + + +def _rank_key(row: dict[str, Any]) -> tuple[int, float]: + """Fewest triggered sessions first (an exact count: the rates share one session total), then the lowest acted share.""" + acted = row["acted"] + share = 0.5 if acted is None or not row["episodes"] else acted / row["episodes"] + return (row["sessions_triggered"], share) + + +def build_report(guidance_name: str, guidance_text: str, sessions: list[Session], *, window: int, + min_sessions: int, rare_max: int, in_use_rate: float) -> dict[str, Any]: + """The prefilter report: every claim with its status, the candidates ranked first.""" + total = len(sessions) + rows = [] + for claim in claims_from_guidance(guidance_text): + t = tally(claim, sessions, window) + status = status_of(claim, t, total, min_sessions=min_sessions, rare_max=rare_max, in_use_rate=in_use_rate) + rows.append(_row(claim, t, status, total)) + candidates = sorted((r for r in rows if r["status"] == CANDIDATE), key=_rank_key) + for rank, row in enumerate(candidates, 1): + row["rank"] = rank + others = sorted((r for r in rows if r["status"] != CANDIDATE), key=lambda r: STATUSES.index(r["status"])) + return { + "format": "session_prefilter", + "format_version": 1, + "guidance": guidance_name, + "sessions": total, + "parameters": {"window": window, "min_sessions": min_sessions, "rare_max": rare_max, + "in_use_rate": in_use_rate}, + "note": NOTE, + "claims": candidates + others, + } + + +def render_table(report: dict[str, Any]) -> str: + head = f"{'rank':>4} {'status':<20} {'trig.sess':>10} {'episodes':>8} {'acted':>7} {'corr.':>6} claim" + lines = [head] + for row in report["claims"]: + acted = "n/a" if row["acted"] is None else f"{row['acted']}/{row['episodes']}" + rank = "-" if row["rank"] is None else str(row["rank"]) + trig = f"{row['sessions_triggered']}/{report['sessions']}" + lines.append(f"{rank:>4} {row['status']:<20} {trig:>10} {row['episodes']:>8} {acted:>7} " + f"{row['corrected']:>6} {row['text'][:70]}") + counts = {s: sum(1 for r in report["claims"] if r["status"] == s) for s in STATUSES} + lines.append("") + lines.append(f"{report['sessions']} session(s), {len(report['claims'])} claim(s): " + + ", ".join(f"{s} {n}" for s, n in counts.items() if n)) + lines.append(report["note"]) + return "\n".join(lines) diff --git a/dokimi/dokimi/context/session_log.py b/dokimi/dokimi/context/session_log.py new file mode 100644 index 000000000..4c3ac0b03 --- /dev/null +++ b/dokimi/dokimi/context/session_log.py @@ -0,0 +1,146 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A Claude Code session log (``~/.claude/projects//.jsonl``) -> prompt and tool events. + +One JSON object per line. A prompt the user typed is a ``user`` record with text content. +A tool call is a ``tool_use`` block of an ``assistant`` record; its ``tool_result`` comes back +in a ``user`` record this reader skips. Records Claude Code injects itself (``isMeta``, such as +CLAUDE.md echoed into the context) are not prompts. A subagent's brief (a ``user`` record with +``isSidechain``) is the parent agent's words, not the user's, so it is not a prompt either; the +subagent's own tool calls still count as the agent acting. + +Tool inputs are kept as text for matching, except the bodies of an edit or a write, which would +echo file content into the match. Nothing here calls a model. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# Input keys whose value is file content, not the agent's intent: left out of an event's text. +_CONTENT_KEYS = frozenset({"old_string", "new_string", "content", "new_source", "edits", + "file_text", "old_str", "new_str"}) # the last three: Goose's editor tools +_PATH_KEYS = ("file_path", "path", "notebook_path") +# Markup a harness wraps around text it injects into a user turn; not the user's words. +_INJECTED = re.compile(r"<(system-reminder|command-[a-z-]+|local-command-[a-z-]+)>.*?", re.S) +_ERROR_LIMIT = 20 + + +class SessionError(ValueError): + """The file is not a session log this module can read.""" + + +@dataclass(frozen=True) +class Event: + """One thing that happened in a session, in file order.""" + + kind: str # "prompt": the user typed it; "tool": the agent called a tool + text: str # a prompt's text, or a tool call's string inputs (no file bodies) + command: str = "" # a Bash call's command line + paths: tuple[str, ...] = () # file paths a tool call named + + +@dataclass +class Session: + """The events of one session log, plus the lines that could not be read.""" + + name: str + events: list[Event] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +def read_session(path: Path) -> Session: + try: + raw = path.read_text(encoding="utf-8-sig", errors="replace") + except OSError as exc: + raise SessionError(f"cannot read {path}: {exc}") from exc + session = parse_session(raw, str(path)) + if not session.events: + raise SessionError(f"{path}: no prompt and no tool call; is it a Claude Code session log?") + return session + + +def parse_session(raw: str, name: str = "") -> Session: + session = Session(name=name) + for n, line in enumerate(raw.split("\n"), 1): # not splitlines(): it breaks at a raw U+2028 + if not line.strip(): + continue + try: + rec = json.loads(line) + except (ValueError, RecursionError): # JSONDecodeError, an over-long integer, deep nesting + add_error(session, f"line {n}: not JSON") + continue + if not isinstance(rec, dict): + add_error(session, f"line {n}: not a JSON object") + continue + problem = _shape_problem(rec) + if problem: + add_error(session, f"line {n}: {problem}") + continue + session.events.extend(_events_of(rec)) + return session + + +def prompt_text(raw: str) -> str: + """The user's own words: harness markup removed, edges trimmed.""" + return _INJECTED.sub(" ", raw).strip() + + +def add_error(session: Session, message: str) -> None: + if len(session.errors) < _ERROR_LIMIT: + session.errors.append(message) + + +def _shape_problem(rec: dict[str, Any]) -> str | None: + """What is wrong with a prompt or tool record's message, if anything. Other record types (summaries, and + tool results, which are user records of tool_result blocks) are recognized and not checked here.""" + if rec.get("type") not in ("user", "assistant"): + return None + message = rec.get("message") + if message is None: + return None + if not isinstance(message, dict): + return "message is not an object" + content = message.get("content") + if content is not None and not isinstance(content, (str, list)): + return "message content is not text or a list of blocks" + return None + + +def _events_of(rec: dict[str, Any]) -> list[Event]: + if rec.get("isMeta"): + return [] + message = rec.get("message") + content = message.get("content") if isinstance(message, dict) else None + if rec.get("type") == "user": + if rec.get("isSidechain"): + return [] + text = prompt_text(_text_of(content)) + return [Event("prompt", text)] if text else [] + if rec.get("type") == "assistant" and isinstance(content, list): + return [tool_event(b.get("input")) for b in content + if isinstance(b, dict) and b.get("type") == "tool_use"] + return [] + + +def _text_of(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join(b.get("text", "") for b in content + if isinstance(b, dict) and b.get("type") == "text" and isinstance(b.get("text"), str)) + return "" + + +def tool_event(tool_input: Any) -> Event: + args = tool_input if isinstance(tool_input, dict) else {} + strings = [v for k, v in args.items() if k not in _CONTENT_KEYS and isinstance(v, str)] + raw_command = args.get("command") + command = raw_command if isinstance(raw_command, str) else "" + paths = tuple(args[k] for k in _PATH_KEYS if isinstance(args.get(k), str)) + return Event("tool", " ".join(strings), command=command, paths=paths) diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 0f2d53b27..1e8d45996 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -121,6 +121,7 @@ nav: - subscription-backends.md - answer-formats.md - context-window.md + - session-prefilter.md - Content novelty: - skill-novelty.md - skill-novelty-walkthrough.md diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index 1350cae82..fc63a6e9f 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -223,5 +223,8 @@ --work-output-dir --work-repeats -o +--min-sessions # context prefilter threshold: no registry concept yet +--rare-max # context prefilter threshold: no registry concept yet +--in-use-rate # context prefilter threshold: no registry concept yet --graph # eval coverage: the frozen need graph input, a file no registry concept names yet (#331) --min # eval coverage gate floor, [METRIC=]X like --timeout [PHASE=]SEC (#331) diff --git a/dokimi/tests/test_context_goose.py b/dokimi/tests/test_context_goose.py new file mode 100644 index 000000000..447cbc21b --- /dev/null +++ b/dokimi/tests/test_context_goose.py @@ -0,0 +1,207 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Goose session stores read by `context prefilter`: synthetic SQLite stores under ``tmp_path``. + +The table and column names follow the block/goose source (session storage and message +serialization); no real store is read. +""" + +from __future__ import annotations + +import json +import sqlite3 + +import pytest + +from dokimi.cli import build_parser +from dokimi.context.goose_log import is_goose_store, read_goose_store +from dokimi.context.session_log import SessionError + +SCHEMA = """ +CREATE TABLE sessions (id TEXT PRIMARY KEY, created_at TEXT NOT NULL); +CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + content_json TEXT NOT NULL, + created_timestamp INTEGER NOT NULL, + message_id TEXT, + metadata_json TEXT +); +""" +GUIDE = "- Keep the changelog fragment current.\n" + + +def _text(words): + return [{"type": "text", "text": words}] + + +def _call(command=None, path=None): + args = {} + if command is not None: + args["command"] = command + if path is not None: + args["path"] = path + return [{"type": "toolRequest", "id": "c1", + "toolCall": {"status": "success", "value": {"name": "developer__shell", "arguments": args}}}] + + +def _result(): + return [{"type": "toolResponse", "id": "c1", "toolResult": {"status": "success", "value": {}}}] + + +def _store(tmp_path, sessions, name="sessions.db"): + """``sessions`` maps a session id to its rows: (role, content, metadata or None), in order.""" + db = tmp_path / name + conn = sqlite3.connect(db) + conn.executescript(SCHEMA) + for n, (sid, rows) in enumerate(sessions.items()): + conn.execute("INSERT INTO sessions VALUES (?, ?)", (sid, f"2026-10-30T10:0{n}:00Z")) + for ts, (role, content, meta) in enumerate(rows, 1): + conn.execute( + "INSERT INTO messages (session_id, role, content_json, created_timestamp, metadata_json) " + "VALUES (?, ?, ?, ?, ?)", + (sid, role, json.dumps(content), ts, json.dumps(meta) if meta is not None else None)) + conn.commit() + conn.close() + return db + + +def test_each_row_is_a_session_and_a_session_without_a_prompt_or_call_is_dropped(tmp_path): + db = _store(tmp_path, { + "s0": [("user", _result(), None)], + "s1": [("user", _text("keep the changelog fragment current"), None), + ("assistant", _call(command="make ci-local"), None), + ("user", _result(), None)], + }) + (session,) = read_goose_store(db) + assert session.name == f"{db}#s1" + assert [(e.kind, e.text) for e in session.events] == [ + ("prompt", "keep the changelog fragment current"), ("tool", "make ci-local")] + assert session.events[1].command == "make ci-local" + + +def test_a_session_of_unreadable_messages_only_is_not_counted(tmp_path): + db = _store(tmp_path, {"s0": [("user", _text("push the branch now"), None)], "s1": []}) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO messages (session_id, role, content_json, created_timestamp) " + "VALUES ('s1', 'user', 'not json', 1)") + conn.commit() + conn.close() + sessions = read_goose_store(db) + assert [s.name for s in sessions] == [f"{db}#s0"] + + +def test_tool_arguments_come_from_the_tool_call_value_and_name_a_path(tmp_path): + db = _store(tmp_path, {"s1": [("assistant", _call(path="/w/docs/guide.md"), None)]}) + (session,) = read_goose_store(db) + assert session.events[0].paths == ("/w/docs/guide.md",) + + +def test_a_message_the_user_cannot_see_is_not_a_prompt(tmp_path): + db = _store(tmp_path, {"s1": [ + ("user", _text("note for the agent only"), {"userVisible": False, "agentVisible": True}), + ("user", _text("the user's own words"), {"userVisible": True, "agentVisible": True}), + ("user", _text("a prompt without metadata"), None), + ]}) + (session,) = read_goose_store(db) + assert [e.text for e in session.events] == ["the user's own words", "a prompt without metadata"] + + +def test_a_message_with_bad_content_is_reported_and_the_rest_still_read(tmp_path): + db = _store(tmp_path, {"s1": [("user", _text("push the branch now"), None)]}) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO messages (session_id, role, content_json, created_timestamp) " + "VALUES ('s1', 'user', 'not json', 9)") + conn.commit() + conn.close() + (session,) = read_goose_store(db) + assert [e.text for e in session.events] == ["push the branch now"] + assert len(session.errors) == 1 and session.errors[0].endswith("content is not JSON") + + +def test_a_missing_store_is_refused_and_nothing_is_created(tmp_path): + missing = tmp_path / "none.db" + with pytest.raises(SessionError, match="cannot read"): + read_goose_store(missing) + assert not missing.exists() + + +def test_a_file_that_is_not_a_store_is_refused(tmp_path): + plain = tmp_path / "plain.db" + plain.write_text("hello, not sqlite\n", encoding="utf-8") + assert not is_goose_store(plain) + with pytest.raises(SessionError, match="not a Goose sessions store"): + read_goose_store(plain) + + +def test_a_store_with_no_prompt_or_tool_call_is_refused(tmp_path): + db = _store(tmp_path, {"s1": [("user", _result(), None)]}) + with pytest.raises(SessionError, match="no prompt and no tool call"): + read_goose_store(db) + + +def test_the_command_reads_a_store_and_a_claude_log_together(tmp_path, capsys): + guide = tmp_path / "AGENTS.md" + guide.write_text(GUIDE, encoding="utf-8") + db = _store(tmp_path, {"s1": [("user", _text("fix the readme typo"), None)]}) + claude = tmp_path / "claude.jsonl" + claude.write_text(json.dumps({"type": "user", "message": {"role": "user", "content": "fix the readme"}}) + "\n", + encoding="utf-8") + args = build_parser().parse_args(["context", "prefilter", str(guide), str(db), str(claude), + "--min-sessions", "1", "--json"]) + args.func(args) + assert json.loads(capsys.readouterr().out)["sessions"] == 2 + + +def test_a_user_message_of_several_text_blocks_is_one_prompt(tmp_path): + db = _store(tmp_path, {"s1": [("user", [{"type": "text", "text": "update the changelog fragment"}, + {"type": "text", "text": "no, update the changelog fragment first"}], + None)]}) + (session,) = read_goose_store(db) + assert [e.kind for e in session.events] == ["prompt"] + + +def test_an_unreadable_only_session_is_dropped_from_the_corpus_and_its_diagnostic_is_kept(tmp_path): + from dokimi.commands.context import _read_sessions + db = _store(tmp_path, {"s0": [("user", _text("push the branch now"), None)], "s1": []}) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO messages (session_id, role, content_json, created_timestamp) " + "VALUES ('s1', 'user', 'not json', 1)") + conn.commit() + conn.close() + dropped: list = [] + sessions = read_goose_store(db, dropped) + assert [s.name for s in sessions] == [f"{db}#s0"] + assert len(dropped) == 1 and dropped[0].errors[0].endswith("content is not JSON") + loaded, failed = _read_sessions([str(db)]) + assert len(loaded) == 1 and not failed + + +def test_a_goose_session_with_a_bad_message_reports_it_on_the_command_line(tmp_path, capsys): + from dokimi.commands.context import _read_sessions + db = _store(tmp_path, {"s0": [("user", _text("push the branch now"), None)], "s1": []}) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO messages (session_id, role, content_json, created_timestamp) " + "VALUES ('s1', 'user', 'not json', 1)") + conn.commit() + conn.close() + _read_sessions([str(db)]) + assert "content is not JSON" in capsys.readouterr().err + + +def test_a_message_whose_content_is_an_object_is_reported_and_the_rest_still_reads(tmp_path): + db = _store(tmp_path, {"s1": [("user", _text("push the branch now"), None), + ("user", _result(), None)]}) + conn = sqlite3.connect(db) + conn.execute("INSERT INTO messages (session_id, role, content_json, created_timestamp) " + "VALUES ('s1', 'user', '{\"type\": \"text\", \"text\": \"x\"}', 9)") + conn.execute("INSERT INTO messages (session_id, role, content_json, created_timestamp) " + "VALUES ('s1', 'user', 'null', 10)") + conn.commit() + conn.close() + (session,) = read_goose_store(db) + assert [e.text for e in session.events] == ["push the branch now"] + assert len(session.errors) == 2 + assert all(err.endswith("content is not a list of blocks") for err in session.errors) diff --git a/dokimi/tests/test_context_prefilter.py b/dokimi/tests/test_context_prefilter.py new file mode 100644 index 000000000..7f1a676ce --- /dev/null +++ b/dokimi/tests/test_context_prefilter.py @@ -0,0 +1,900 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`context prefilter`: guidance rules matched against Claude Code session logs, candidates for ablation. + +Every session here is a small synthetic fixture written under ``tmp_path``; no real transcript is read. +""" + +from __future__ import annotations + +import json +import re + +import pytest + +from dokimi.cli import build_parser +from dokimi.context.prefilter import ( + CANDIDATE, + IN_USE, + INSUFFICIENT, + KEPT, + RARE, + STATUSES, + Tally, + build_report, + claims_from_guidance, + render_table, + status_of, +) +from dokimi.context.session_log import SessionError, parse_session, read_session + +GUIDE = """# Guide + +- Run `make ci-local` before pushing a branch. +- Keep the changelog fragment current. +- Check the onboarding wiki page for newcomers. +- Use `ruff format` when touching Python files. + +| Command | Purpose | +| --- | --- | +| `pytest tests/test_x.py` | Run one file | + +```bash +echo "this line is code and no claim" +``` +""" + +CHANGELOG = "Keep the changelog fragment current" +CI = "Run `make ci-local` before pushing a branch" + + +def _prompt(text, **extra): + return {"type": "user", "message": {"role": "user", "content": text}, **extra} + + +def _tool(name, **args): + return {"type": "assistant", "message": {"role": "assistant", + "content": [{"type": "tool_use", "id": "t", "name": name, "input": args}]}} + + +def _session(*records, name="s"): + return parse_session("\n".join(json.dumps(r) for r in records), name) + + +def _write(path, records): + path.write_text("\n".join(json.dumps(r) for r in records) + "\n", encoding="utf-8") + return path + + +def _row(report, text): + return next(r for r in report["claims"] if text in r["text"]) + + +def _report(sessions, guide=GUIDE, **kw): + params = {"window": 10, "min_sessions": 1, "rare_max": 0, "in_use_rate": 0.2} + params.update(kw) + return build_report("AGENTS.md", guide, sessions, **params) + + +# --- session_log: what becomes an event -------------------------------------------------------- + +def test_prompts_are_typed_text_and_injected_records_are_not(): + session = _session( + _prompt("CLAUDE.md echo: keep the changelog fragment current", isMeta=True), + _prompt("keep the changelog please ship it"), + _prompt("the subagent brief", isSidechain=True), + {"type": "user", "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t", "content": "ok"}]}}, + ) + assert [e.text for e in session.events] == ["please ship it"] + + +def test_tool_call_keeps_command_and_paths_and_drops_file_bodies(): + session = _session(_tool("Edit", file_path="/w/docs/guide.md", old_string="keep the changelog", + new_string="rewritten body", description="tidy")) + (event,) = session.events + assert event.kind == "tool" + assert event.paths == ("/w/docs/guide.md",) + assert "tidy" in event.text and "/w/docs/guide.md" in event.text + assert "keep the changelog" not in event.text and "rewritten body" not in event.text + + +def test_bash_command_is_its_own_field_and_a_subagent_call_still_counts(): + record = _tool("Bash", command="make ci-local --offline") + record["isSidechain"] = True + (event,) = _session(record).events + assert event.command == "make ci-local --offline" + + +def test_a_bad_line_is_reported_and_the_rest_still_reads(): + raw = "not json\n" + json.dumps(_prompt("hello there")) + "\n" + session = parse_session(raw, "x") + assert [e.text for e in session.events] == ["hello there"] + assert session.errors == ["line 1: not JSON"] + + +def test_a_line_separator_inside_a_prompt_does_not_split_the_record(): + raw = json.dumps(_prompt("keep the
changelog fragment"), ensure_ascii=False) + "\n" + session = parse_session(raw, "x") + assert session.errors == [] and [e.text for e in session.events] == ["keep the
changelog fragment"] + + +def test_a_line_nested_too_deep_to_parse_is_reported_and_skipped(): + raw = "[" * 100_000 + "\n" + json.dumps(_prompt("still here")) + "\n" + session = parse_session(raw, "x") + assert [e.text for e in session.events] == ["still here"] + assert session.errors == ["line 1: not JSON"] + + +def test_read_session_refuses_a_file_it_cannot_read_or_that_holds_no_event(tmp_path): + with pytest.raises(SessionError, match="cannot read"): + read_session(tmp_path / "missing.jsonl") + empty = _write(tmp_path / "empty.jsonl", [{"type": "summary", "summary": "x"}]) + with pytest.raises(SessionError, match="no prompt and no tool call"): + read_session(empty) + + +# --- claims: what a guidance file's rules are -------------------------------------------------- + +def test_claims_are_rule_lines_and_skip_headings_and_code(): + texts = [c.text for c in claims_from_guidance(GUIDE)] + assert any(CI in t for t in texts) and any("ruff format" in t for t in texts) + assert not any(t.startswith("# ") or "this line is code" in t for t in texts) + assert not any(t == "Guide" for t in texts) + + +def test_backticked_names_split_into_commands_and_paths(): + claims = {c.text: c for c in claims_from_guidance(GUIDE)} + ci = next(c for t, c in claims.items() if CI in t) + assert ci.commands == ("make ci-local",) and ci.paths == () + table_row = next(c for t, c in claims.items() if "pytest" in t) + assert table_row.commands == ("pytest tests/test_x.py",) and table_row.paths == () + changelog = next(c for c in claims_from_guidance("- Keep `changelog.d/` current.\n")) + assert changelog.paths == ("changelog.d/",) and changelog.commands == () + + +def test_a_rule_with_a_negation_carries_its_protection_tag(): + claims = claims_from_guidance("- Never commit secrets to the repository.\n") + assert claims and claims[0].protection + + +# --- matching: trigger, acted, corrected --------------------------------------------------------- + +def test_an_episode_counts_once_inside_its_window(): + session = _session(_prompt("update the changelog fragment"), + _prompt("also the changelog fragment for the second part")) + row = _row(_report([session]), CHANGELOG) + assert row["episodes"] == 1 and row["sessions_triggered"] == 1 + + +def test_claim_is_acted_on_when_the_named_command_runs_near_the_trigger(): + acting = _session(_prompt("push the branch now"), _tool("Bash", command="make ci-local"), + _tool("Bash", command="git push origin b"), name="acting") + idle = _session(_prompt("push the branch now"), _tool("Bash", command="git push origin b"), name="idle") + row = _row(_report([acting, idle]), CI) + assert row["episodes"] == 2 and row["acted"] == 1 and row["acted_rate"] == 0.5 + + +def test_a_longer_command_is_not_the_named_one(): + session = _session(_prompt("push the branch now"), _tool("Bash", command="make ci-local-extra")) + assert _row(_report([session]), CI)["acted"] == 0 + + +def test_a_user_pushback_after_the_trigger_is_a_correction(): + corrected = _session(_prompt("push the branch"), _prompt("no, the branch must not go before ci")) + agreed = _session(_prompt("push the branch"), _prompt("push the branch again, thanks")) + row = _row(_report([corrected, agreed]), CI) + assert row["corrected"] == 1 and row["correction_rate"] == 0.5 + + +def test_a_path_matches_from_a_segment_start_not_inside_a_longer_name(): + rule = "- Update `a.md` before the release.\n" + inside = _session(_prompt("update the release"), _tool("Edit", file_path="/w/data.md")) + named = _session(_prompt("update the release"), _tool("Edit", file_path="/w/docs/a.md")) + assert _row(_report([inside, named], rule), "Update")["acted"] == 1 + + +def test_a_file_name_does_not_match_a_longer_extension_or_a_backup_of_it(): + rule = "- Update `a.md` before the release.\n" + longer = _session(_prompt("update the release"), _tool("Edit", file_path="/w/a.mdx")) + backup = _session(_prompt("update the release"), _tool("Edit", file_path="/w/a.md.bak")) + assert _row(_report([longer, backup], rule), "Update")["acted"] == 0 + + +def test_a_directory_name_matches_the_entries_under_it(): + rule = "- Keep `changelog.d/` current.\n" + inside = _session(_prompt("keep the changelog current"), _tool("Write", file_path="/w/changelog.d/x.md")) + assert _row(_report([inside], rule), "Keep")["acted"] == 1 + + +def test_a_claim_without_a_named_command_is_not_measured_as_acted(): + session = _session(_prompt("keep the changelog fragment current")) + row = _row(_report([session]), CHANGELOG) + assert row["acted"] is None and "acted_not_measurable" in row["flags"] + + +def test_one_shared_word_of_a_three_word_rule_is_not_a_trigger(): + partial = _session(_prompt("keep the readme current"), _prompt("fix the readme typo")) + assert _row(_report([partial]), CHANGELOG)["sessions_triggered"] == 0 + + +def test_a_memory_echo_of_the_rule_is_not_a_trigger(): + echo = _session(_prompt("Keep the changelog fragment current", isMeta=True), _prompt("fix the readme")) + assert _row(_report([echo]), CHANGELOG)["sessions_triggered"] == 0 + + +# --- status and ranking -------------------------------------------------------------------------- + +def _ranking_corpus(): + sessions = [] + for i in range(6): + events = [_prompt("fix the readme typo")] + if i < 4: + events.append(_prompt("keep the changelog fragment current")) + if i < 2: + events.append(_prompt("touching python files")) + if i == 5: + events.append(_prompt("release the standup notes")) + sessions.append(_session(*events, name=f"s{i}")) + return sessions + + +GUIDE_RANK = """- Keep the changelog fragment current. +- Use `ruff format` when touching Python files. +- Never commit secrets to the repository. +- Check the onboarding wiki page for newcomers. +- Mention the release train in standup notes. +""" + + +def test_statuses_split_in_use_rare_protected_and_candidates(): + # Triggered in 4, 2, 0 and 1 of 6 sessions; rare_max=3 makes 2 and 1 rare, and 4 is over the in-use rate. + report = _report(_ranking_corpus(), GUIDE_RANK, rare_max=3, in_use_rate=0.5) + assert _row(report, CHANGELOG)["status"] == IN_USE + assert _row(report, "Use `ruff")["status"] == RARE + assert _row(report, "Mention the release")["status"] == RARE + assert _row(report, "Never commit")["status"] == KEPT + assert _row(report, "Check the")["status"] == CANDIDATE + + +def test_rare_and_protected_rules_are_listed_but_never_ranked(): + report = _report(_ranking_corpus(), GUIDE_RANK, rare_max=3, in_use_rate=0.5) + assert all(r["rank"] is None for r in report["claims"] if r["status"] != CANDIDATE) + assert _row(report, "Check the")["rank"] == 1 # the only candidate left: zero triggers + + +def test_candidates_are_ranked_by_trigger_rate_lowest_first(): + report = _report(_ranking_corpus(), GUIDE_RANK, rare_max=0, in_use_rate=0.5) + assert [_row(report, t)["rank"] for t in ("Check the", "Mention the release", "Use `ruff")] == [1, 2, 3] + assert _row(report, CHANGELOG)["status"] == IN_USE and _row(report, CHANGELOG)["rank"] is None + + +def test_a_protected_rule_is_kept_even_with_no_trigger_at_all(): + report = _report(_ranking_corpus(), GUIDE_RANK, rare_max=3, in_use_rate=0.5) + row = _row(report, "Never commit") + assert row["status"] == KEPT and row["rank"] is None and row["sessions_triggered"] == 0 + + +def test_too_few_sessions_rank_nothing(): + report = _report(_ranking_corpus()[:2], GUIDE_RANK, min_sessions=5) + assert {r["status"] for r in report["claims"]} == {INSUFFICIENT, KEPT} + assert all(r["rank"] is None for r in report["claims"]) + + +def test_the_wording_never_says_cut_or_remove(): + report = _report(_ranking_corpus(), GUIDE_RANK, rare_max=3, in_use_rate=0.5) + text = json.dumps(report) + render_table(report) + assert not re.search(r"\b(cut|remove|delete|drop)\b", text, re.I) + assert set(r["status"] for r in report["claims"]) <= set(STATUSES) + + +def test_the_table_lists_each_claim_and_states_it_is_a_prefilter(): + report = _report(_ranking_corpus(), GUIDE_RANK, rare_max=3, in_use_rate=0.5) + table = render_table(report) + assert "candidate_for_ablation" in table and "kept_protected" in table + assert "not verdicts" in table + + +# --- command line ------------------------------------------------------------------------------- + +def test_the_command_reads_a_quoted_glob_and_prints_json(tmp_path, capsys): + guide = tmp_path / "AGENTS.md" + guide.write_text(GUIDE, encoding="utf-8") + logs = tmp_path / "logs" + logs.mkdir() + for i in range(3): + _write(logs / f"s{i}.jsonl", [_prompt("fix the readme typo")]) + args = build_parser().parse_args(["context", "prefilter", str(guide), f"{logs}/*.jsonl", + "--min-sessions", "1", "--json"]) + args.func(args) + report = json.loads(capsys.readouterr().out) + assert report["format"] == "session_prefilter" and report["format_version"] == 1 + assert report["sessions"] == 3 + + +def test_the_command_prints_a_table_and_still_reports_a_bad_session(tmp_path, capsys): + guide = tmp_path / "AGENTS.md" + guide.write_text(GUIDE, encoding="utf-8") + good = _write(tmp_path / "good.jsonl", [_prompt("push the branch now")]) + bad = tmp_path / "bad.jsonl" + bad.write_text("not json at all\n", encoding="utf-8") + args = build_parser().parse_args(["context", "prefilter", str(guide), str(good), str(bad), + "--min-sessions", "1"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + out, err = capsys.readouterr() + assert exc.value.code == 1 + assert "rank" in out and "1 session(s)" in out + assert "no prompt and no tool call" in err + + +def test_the_command_refuses_a_guidance_file_it_cannot_read(tmp_path): + args = build_parser().parse_args(["context", "prefilter", str(tmp_path / "none.md"), str(tmp_path / "x.jsonl")]) + with pytest.raises(SystemExit, match="cannot read"): + args.func(args) + + + +# --- review fixes: keyless rules, separators, shell commands, corrections, ranking, sources -------- + +def test_a_rule_of_named_commands_only_is_kept_and_triggers_on_its_command(): + guide = "- Run `pytest`.\n- Do not use `sudo`.\n" + session = _session(_prompt("the tests: pytest"), _tool("Bash", command="uv run pytest -q")) + rows = _report([session], guide) + run = _row(rows, "Run `pytest`") + assert run["episodes"] == 1 and run["acted"] == 1 + assert _row(rows, "sudo")["status"] == KEPT + + +def test_windows_separators_match_a_path_and_a_backslash_directory_is_a_path(): + rule = "- Keep `changelog.d/` current.\n- Update `docs/guide.md` before release.\n" + write = _session(_prompt("keep the changelog current"), _tool("Write", file_path="C:\\w\\changelog.d\\x.md")) + assert _row(_report([write], rule), "Keep")["acted"] == 1 + edit = _session(_prompt("update the release"), _tool("Edit", file_path="C:\\work\\docs\\guide.md")) + assert _row(_report([edit], rule), "Update")["acted"] == 1 + (dir_claim,) = claims_from_guidance("- Keep `C:\\repo\\docs\\` clean.\n") + assert dir_claim.paths == ("C:\\repo\\docs\\",) and dir_claim.commands == () + + +def test_a_shell_command_that_writes_a_named_directory_counts_as_acting(): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command="printf 'entry' > changelog.d/x.md")) + assert _row(_report([session], rule), "Keep")["acted"] == 1 + + +@pytest.mark.parametrize("command", ["echo 'make ci-local'", "grep -F 'make ci-local' AGENTS.md", "# make ci-local", + "echo 'a; make ci-local'", "ls # x; make ci-local"]) +def test_a_mention_of_the_command_is_not_an_action(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 0 + + +@pytest.mark.parametrize("command", ["make ci-local", "uv run make ci-local", "cd x && make ci-local", + "make lint & make ci-local", "timeout 600 make ci-local", "make ci-local"]) +def test_a_command_run_directly_after_a_runner_or_in_a_chain_is_an_action(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 1 + + +@pytest.mark.parametrize("command", ["make ci-local.debug", "make ci-local/x", "make ci-local=1"]) +def test_a_longer_target_of_the_named_command_is_not_an_action(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 0 + + +def test_a_quoted_argument_of_the_named_command_is_an_action(): + rule = "- Run `pytest tests/test_x.py` before pushing a branch.\n" + session = _session(_prompt("push the branch now"), _tool("Bash", command='pytest "tests/test_x.py" -q')) + assert _row(_report([session], rule), "Run")["acted"] == 1 + + +@pytest.mark.parametrize("command", ["echo 'changelog.d/x.md'", "ls # changelog.d/x.md", "echo changelog.d/x.md"]) +def test_a_path_printed_or_commented_by_a_shell_command_is_not_acted_on(command): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command=command)) + assert _row(_report([session], rule), "Keep")["acted"] == 0 + + +def test_a_quoted_separator_or_hash_in_a_path_argument_is_kept_as_text(): + rule = "- Keep `ab.py` current.\n" + session = _session(_prompt("keep ab.py current"), _tool("Bash", command='cat "a#b.py"')) + assert _row(_report([session], rule), "Keep")["acted"] == 0 + + +def test_a_path_read_by_a_shell_command_is_acted_on(): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command="cat changelog.d/x.md")) + assert _row(_report([session], rule), "Keep")["acted"] == 1 + + +def test_a_correction_is_read_from_the_next_prompt_only(): + later = _session(_prompt("push the branch"), _prompt("thanks, continue"), + _prompt("no, the branch must not go before ci")) + assert _row(_report([later]), CI)["corrected"] == 0 + + +def test_goose_edit_bodies_are_not_event_text(): + from dokimi.context.session_log import tool_event + event = tool_event({"path": "/w/README.md", "file_text": "Keep the changelog fragment current", + "old_str": "changelog", "new_str": "changelog fragment"}) + assert "changelog" not in event.text and event.paths == ("/w/README.md",) + + +def test_candidates_with_equal_rounded_rates_rank_by_exact_count(): + from dokimi.context.prefilter import _rank_key + more_triggered = {"sessions_triggered": 3, "trigger_rate": 0.001, "episodes": 3, "acted": 0, "acted_rate": 0.0} + fewer_triggered = {"sessions_triggered": 2, "trigger_rate": 0.001, "episodes": 2, "acted": 2, "acted_rate": 1.0} + assert min([more_triggered, fewer_triggered], key=_rank_key) is fewer_triggered + + +def test_a_repeated_or_aliased_source_is_read_once(tmp_path): + from dokimi.commands.context import _read_sessions + log = _write(tmp_path / "s.jsonl", [_prompt("push the branch now")]) + alias = tmp_path / "alias.jsonl" + alias.symlink_to(log) + sessions, failed = _read_sessions([str(log)] * 5 + [str(alias), f"{tmp_path}/*.jsonl"]) + assert len(sessions) == 1 and not failed + + +@pytest.mark.parametrize("flag,value", [("--rare-max", "-1"), ("--in-use-rate", "nan"), ("--in-use-rate", "inf"), + ("--in-use-rate", "1.5"), ("--in-use-rate", "-0.1")]) +def test_the_thresholds_refuse_values_outside_their_domain(tmp_path, flag, value): + guide = tmp_path / "AGENTS.md" + guide.write_text(GUIDE, encoding="utf-8") + log = _write(tmp_path / "s.jsonl", [_prompt("push the branch now")]) + with pytest.raises(SystemExit): + build_parser().parse_args(["context", "prefilter", str(guide), str(log), flag, value]) + + +def test_the_thresholds_accept_their_edge_values(tmp_path): + guide = tmp_path / "AGENTS.md" + guide.write_text(GUIDE, encoding="utf-8") + log = _write(tmp_path / "s.jsonl", [_prompt("push the branch now")]) + args = build_parser().parse_args(["context", "prefilter", str(guide), str(log), + "--rare-max", "0", "--in-use-rate", "1"]) + assert args.rare_max == 0 and args.in_use_rate == 1.0 + + +# --- review round 3: quoting, heredocs, bare names, boundaries, shapes ------------------------- + +@pytest.mark.parametrize("command", ["echo '> changelog.d/x.md'", 'echo "a > changelog.d/x.md"', + "printf '%s' '>>changelog.d/x'", "git commit -m '> changelog.d/x.md'"]) +def test_a_quoted_redirection_sign_is_text_not_a_redirect(command): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command=command)) + assert _row(_report([session], rule), "Keep")["acted"] == 0 + + +def test_a_redirect_after_quoted_text_is_still_a_write_to_its_target(): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), + _tool("Bash", command="echo 'a > b' > changelog.d/x.md")) + assert _row(_report([session], rule), "Keep")["acted"] == 1 + + +def test_a_heredoc_body_naming_the_command_is_text_not_a_run(): + session = _session(_prompt("push the branch now"), + _tool("Bash", command="cat > notes.txt <<'EOF'\nmake ci-local\nEOF")) + assert _row(_report([session]), CI)["acted"] == 0 + + +def test_a_heredoc_body_naming_a_path_does_not_act_on_it_but_a_heredoc_target_does(): + rule = "- Keep `changelog.d/` current.\n" + body = _session(_prompt("keep the changelog current"), + _tool("Bash", command="cat > notes.txt <results.txt", "make ci-local changelog.d/x.md\ntrue']) +def test_a_quoted_heredoc_sign_does_not_erase_the_commands_after_it(command): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command=command)) + assert _row(_report([session], rule), "Keep")["acted"] == 1 + + +def test_a_quoted_heredoc_sign_before_the_named_command_still_runs_it(): + session = _session(_prompt("push the branch now"), _tool("Bash", command='echo "<results.txt make ci-local", "> results.txt make ci-local", + "make >results.txt ci-local", "make ci-local 2>&1 >results.txt", + "2>/dev/null make ci-local"]) +def test_a_redirection_among_the_words_of_a_command_does_not_hide_it(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 1 + + +@pytest.mark.parametrize("command", ['git commit -m "add f() {"; make ci-local', + "echo 'g() {'; make ci-local", + 'git commit -m "x" && git log -- "a() {" ; make ci-local']) +def test_a_function_head_inside_quotes_does_not_hide_the_command_after_it(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 1 + + +@pytest.mark.parametrize("command", ["# note f() {\nmake ci-local", "echo ok # g() {\nmake ci-local"]) +def test_a_function_head_in_a_comment_does_not_hide_the_command_after_it(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 1 + + +def test_a_function_body_that_never_closes_does_not_hide_the_rest_of_the_line(): + session = _session(_prompt("push the branch now"), _tool("Bash", command="check_ci() { echo x; make ci-local")) + assert _row(_report([session]), CI)["acted"] == 1 + + +@pytest.mark.parametrize("command", ["make >> results.txt ci-local", "make 2>> results.txt ci-local", + "make >| results.txt ci-local", "make >& results.txt ci-local", + "make <<< word ci-local", "make 2>&1 >> results.txt ci-local"]) +def test_a_multi_character_redirection_takes_its_target_out_of_the_words(command): + session = _session(_prompt("push the branch now"), _tool("Bash", command=command)) + assert _row(_report([session]), CI)["acted"] == 1 + + +# --- round 10: braces in comments, leading redirections, ``>|`` and ``>&`` targets --------------------------- +def test_a_brace_in_a_comment_of_a_function_body_does_not_close_it(): + session = _session(_prompt("push the branch now"), _tool("Bash", command="check_ci() { # }\nmake ci-local\n}")) + assert _row(_report([session]), CI)["acted"] == 0 + + +def test_a_brace_in_a_comment_of_a_function_body_hides_no_path(): + rule = "- Keep `changelog.d/` current.\n" + command = "write() { # }\ncat changelog.d/x.md\n}" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command=command)) + assert _row(_report([session], rule), "Keep")["acted"] == 0 + + +@pytest.mark.parametrize(("command", "acted"), [ + (">notes.txt echo changelog.d/x.md", 0), + ("2>/dev/null printf 'x' changelog.d/x.md", 0), + ("> notes.txt echo changelog.d/x.md", 0), + (">changelog.d/x.md echo hi", 1), # the redirection target is written + ("printf 'entry' >| changelog.d/x.md", 1), + ("echo entry >& changelog.d/x.md", 1), + ("echo entry >|changelog.d/x.md", 1), +]) +def test_leading_and_noclobber_redirections_of_a_printing_command(command, acted): + rule = "- Keep `changelog.d/` current.\n" + session = _session(_prompt("keep the changelog current"), _tool("Bash", command=command)) + assert _row(_report([session], rule), "Keep")["acted"] == acted