From 1d775e2e9347af351c85766882427a1faf6b32f4 Mon Sep 17 00:00:00 2001 From: Christian Boos Date: Sat, 7 Nov 2026 16:00:23 +0100 Subject: [PATCH 1/3] Show a file grown between refresh and render leaving its session row behind A run refreshes the cache, then renders. When a session file grows in between, the render finds its entry rows stale, parses the grown file and writes the rows itself, stamped with the file's new size and mtime. The session row keeps the counts the refresh computed. The run's session page then shows a turn its session row does not count, and the cache calls the file current. The test grows the file right after the run's refresh, cold and over a populated cache, in three output shapes, and asserts that the session page shows the late turn exactly when the session row counts it. It then asserts that a further run with no change brings the row level with a cache built cold from the same files. Today the first assertion fails in all six cases. The second holds only because the render's entry write leaves the pending-refresh flag set, so the next run refreshes the whole project. Co-Authored-By: Claude Opus 5.5 --- test/test_render_snapshot.py | 103 +++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 test/test_render_snapshot.py diff --git a/test/test_render_snapshot.py b/test/test_render_snapshot.py new file mode 100644 index 00000000..1dff5dd9 --- /dev/null +++ b/test/test_render_snapshot.py @@ -0,0 +1,103 @@ +# pyright: reportPrivateUsage=false +"""A run renders what its cache refresh committed. + +A run refreshes the cache, then renders. A session file can grow in +between: the session is still being written. If the render then parses +the grown file and writes its entry rows itself, it also stamps the +file's new size and mtime, but the session row keeps the counts the +refresh computed. The run's pages then show lines its session row does +not count, and the cache calls the file current. Only the pending-refresh +flag the entry write leaves set makes the next run refresh it: without +it, the session row would stay behind until the file is written again. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Tuple + +import pytest + +from claude_code_log import converter +from claude_code_log.cache import CacheManager, get_library_version + +from test.test_date_filtered_runs import ( + SESSIONS, + SHAPES, + _append, + _build_project, + _convert, + _turn, +) + +GROWN = SESSIONS[0] +# The late turn's reply, the last entry of the grown file. +LATE_STAMP = "2026-01-03T10:07:00Z" + + +def _session_row(project: Path, sid: str) -> Tuple[Any, ...]: + data = CacheManager(project, get_library_version()).get_cached_project_data() + assert data is not None + row = data.sessions[sid] + return ( + row.message_count, + row.last_timestamp, + row.total_input_tokens, + row.total_output_tokens, + ) + + +def _grow_after_the_refresh(monkeypatch: pytest.MonkeyPatch, path: Path) -> list[Path]: + """Append a turn to ``path`` once, right after the run's cache refresh.""" + refresh = converter.ensure_fresh_cache_detailed + grown: list[Path] = [] + + def refresh_then_grow(*args: Any, **kwargs: Any) -> Any: + result = refresh(*args, **kwargs) + if not grown: + _append(path, _turn(path.stem, 3, 3, "LATE-TURN")) + grown.append(path) + return result + + monkeypatch.setattr(converter, "ensure_fresh_cache_detailed", refresh_then_grow) + return grown + + +@pytest.mark.parametrize("warm", [False, True], ids=["cold", "warm"]) +@pytest.mark.parametrize("shape", ["html-combined", "md-sessions", "html-paginated"]) +def test_a_file_grown_between_refresh_and_render_is_refreshed_next_run( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, shape: str, warm: bool +) -> None: + project = _build_project(tmp_path) + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) + jsonl = project / f"{GROWN}.jsonl" + if warm: + # The daily shape: a converted project whose session moved on, so + # the refresh is the incremental one over a populated cache. + _convert(project, shape) + _append(jsonl, _turn(GROWN, 2, 2, "WARM-TURN")) + + with monkeypatch.context() as m: + grown = _grow_after_the_refresh(m, jsonl) + _convert(project, shape) + assert grown == [jsonl], "the file never grew between refresh and render" + + # The run's outputs agree: its session page shows the late turn exactly + # when its session row counts it. + ext = SHAPES[shape]["output_format"] + page = (project / f"session-{GROWN}.{ext}").read_text(encoding="utf-8") + assert ("LATE-TURN" in page) == (_session_row(project, GROWN)[1] == LATE_STAMP), ( + f"the page {'shows' if 'LATE-TURN' in page else 'lacks'} the late turn," + f" the session row says {_session_row(project, GROWN)}" + ) + + # Nothing changes now: this run must still bring the row up to the file. + _convert(project, shape) + + # Control: a cache built cold from the same files. + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "control.db")) + _convert(project, shape) + expected = _session_row(project, GROWN) + + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) + assert _session_row(project, GROWN) == expected From 18d249ac357497a42429c5ca67f116335498743a Mon Sep 17 00:00:00 2001 From: Christian Boos Date: Sat, 7 Nov 2026 16:11:59 +0100 Subject: [PATCH 2/3] Render what the cache refresh committed, and write no entry rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run refreshes the cache, then renders. A session file that grew in between had its rows found stale by the render, which parsed the file again and wrote the rows itself, stamped with the file's new size and mtime. The session rows and aggregates kept the refresh's counts, so the run's pages showed lines its session rows did not count, and the cache called the file current: only the pending-refresh flag made the next run look at the file again. Only the refresh writes entry rows now. Every render-side load passes `snapshot=True` to `load_transcript`, through `load_directory_transcripts` and `_load_sessions_partial`: the session-scoped path, the streaming page loads, the full Phase 2 load, `generate_single_session_file` and the TUI's export. In that mode the loader serves `CacheManager.load_snapshot_entries`, the rows the cache holds at the current content version whatever the file's stamp says now, or a verified parsed-entry store hit. It never parses the file and never writes the cache, and it skips a file the cache has no rows for yet. Lines written after the refresh appear on the next run, which finds the file modified since nothing restamped it. In watch mode that is the next tick: the engine stamps the files before it converts. `--no-cache` and single-file mode have no rows and parse as before. The hierarchy's no-cache-data fallback and Codex's wholesale render write their entry rows and session rows from one load, and keep doing both. The TUI now refreshes the cache right before it exports a live session, since the cache it would otherwise render may date from its start. The tests grow a file right after the refresh on each of these paths, including a forced streaming run and a watch engine driven by hand. They spy on the entry-row writers, check that the page shows what the refresh committed and not the late turn, and check that the next run or tick shows the turn and brings the session row level with a cold cache. Each render-mode call site, the snapshot read and the TUI refresh is pinned by a mutation that fails at least one test. The rule is in dev-docs/application_model.md, under the cache (§ 2.3), with the watch tick's lag in § 2.15, the entry store in § 2.16, and the TUI's refresh in § 2.2. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 4 +- claude_code_log/cache.py | 83 ++++++++---- claude_code_log/converter.py | 37 +++++- claude_code_log/tui.py | 7 +- dev-docs/application_model.md | 41 +++++- test/test_render_snapshot.py | 238 ++++++++++++++++++++++++++++++---- 6 files changed, 355 insertions(+), 55 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75f9800c..ab89b723 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -647,7 +647,9 @@ The tool implements a SQLite-based caching system for performance: locked out is retried after the pool rather than failed. A pass compacts an oversized WAL before it starts and truncates the WAL when it ends. A refresh that fails part-way leaves `projects.refresh_pending` set, and the - next run refreshes that project in full. A date-filtered run refreshes the + next run refreshes that project in full. Only the refresh writes entry + rows: a run renders what its refresh committed, so lines written to a + session while it renders appear on the next run. A date-filtered run refreshes the cache as an unfiltered one would and leaves the files it wrote from its window marked stale, so the next unfiltered run re-renders them. Its refresh therefore loads whole files when they changed and the incremental diff --git a/claude_code_log/cache.py b/claude_code_log/cache.py index 9e2126ff..11a24f76 100644 --- a/claude_code_log/cache.py +++ b/claude_code_log/cache.py @@ -1577,13 +1577,7 @@ def load_cached_entries(self, jsonl_path: Path) -> Optional[List[TranscriptEntry if file_id is None: return None - with self._get_connection() as conn: - rows = conn.execute( - "SELECT content FROM messages WHERE file_id = ? ORDER BY timestamp NULLS LAST", - (file_id,), - ).fetchall() - - return [self._deserialize_entry(row) for row in rows] + return self._entries_for_file(file_id, None, None) def load_cached_entries_filtered( self, jsonl_path: Path, from_date: Optional[str], to_date: Optional[str] @@ -1600,28 +1594,69 @@ def load_cached_entries_filtered( if file_id is None: return None - # The renderer's reading of the window: ``filter_messages_by_date`` - # makes the exact cut after this load, so the two must agree on - # what a bound means. - from .utils import parse_date_bounds + return self._entries_for_file(file_id, from_date, to_date) - from_dt, to_dt = parse_date_bounds(from_date, to_date) + def load_snapshot_entries( + self, + jsonl_path: Path, + from_date: Optional[str] = None, + to_date: Optional[str] = None, + ) -> Optional[List[TranscriptEntry]]: + """The entries the cache holds for ``jsonl_path``, however the file + has moved since they were written. + + This is the render's read. A run refreshes the cache, then renders + what the refresh committed: a session file can grow in between, + and parsing it again here would put lines in the pages that the + session rows and aggregates do not count. Lines that land after the + refresh are the next refresh's. So the file's size, mtime and + sub-agent fingerprint are not consulted; only the content version + is, since rows of another version may lack what the renderer reads. + + None when the cache holds no usable rows for the file: it appeared + after the refresh, which leaves it to the next run. + """ + if self._project_id is None: + return None + + with self._get_connection() as conn: + row = conn.execute( + "SELECT id, content_version FROM cached_files" + " WHERE project_id = ? AND file_name = ?", + (self._project_id, jsonl_path.name), + ).fetchone() + + if row is None or row["content_version"] != content_schema_version(): + return None + return self._entries_for_file(row["id"], from_date, to_date) - # Build query with SQL-based filtering + def _entries_for_file( + self, file_id: int, from_date: Optional[str], to_date: Optional[str] + ) -> List[TranscriptEntry]: + """A cached file's entries, in timestamp order, within the window if + one is given.""" sql = "SELECT content FROM messages WHERE file_id = ?" params: List[Any] = [file_id] - if from_dt: - # Same format as stored timestamps (``_normalize_timestamp``) - from_bound = from_dt.strftime("%Y-%m-%dT%H:%M:%SZ") - # Include entries with NULL timestamp (like summaries) OR within date range - sql += " AND (timestamp IS NULL OR timestamp >= ?)" - params.append(from_bound) - - if to_dt: - to_bound = to_dt.strftime("%Y-%m-%dT%H:%M:%SZ") - sql += " AND (timestamp IS NULL OR timestamp <= ?)" - params.append(to_bound) + if from_date or to_date: + # The renderer's reading of the window: ``filter_messages_by_date`` + # makes the exact cut after this load, so the two must agree on + # what a bound means. + from .utils import parse_date_bounds + + from_dt, to_dt = parse_date_bounds(from_date, to_date) + + if from_dt: + # Same format as stored timestamps (``_normalize_timestamp``) + from_bound = from_dt.strftime("%Y-%m-%dT%H:%M:%SZ") + # Include entries with NULL timestamp (like summaries) OR within date range + sql += " AND (timestamp IS NULL OR timestamp >= ?)" + params.append(from_bound) + + if to_dt: + to_bound = to_dt.strftime("%Y-%m-%dT%H:%M:%SZ") + sql += " AND (timestamp IS NULL OR timestamp <= ?)" + params.append(to_bound) sql += " ORDER BY timestamp NULLS LAST" diff --git a/claude_code_log/converter.py b/claude_code_log/converter.py index 58b9373f..61fa1033 100644 --- a/claude_code_log/converter.py +++ b/claude_code_log/converter.py @@ -480,10 +480,17 @@ def load_transcript( _loaded_files: Optional[set[Path]] = None, _meta_maps: Optional[dict[Path, dict[str, str]]] = None, entry_store: "Optional[ParsedEntryStore]" = None, + snapshot: bool = False, ) -> list[TranscriptEntry]: """Load and parse JSONL transcript file, using cache if available. Args: + snapshot: The render's load, after this run's cache refresh: serve + the entries the refresh committed (``load_snapshot_entries``), + even when the file has grown since, and never parse the file + or write the cache. A file the cache has no rows for yet is + left to the next run. Without a cache manager it changes + nothing: there are no rows to disagree with. entry_store: Optional per-conversion store of entries this conversion has already materialised (``entry_store.py``). Consulted ahead of the cache, which it saves a @@ -514,6 +521,15 @@ def load_transcript( if not silent: print(f"Loading {jsonl_path} from this run's parsed entries...") return held + if snapshot and cache_manager is not None: + committed = cache_manager.load_snapshot_entries(jsonl_path, from_date, to_date) + if committed is None: + if not silent: + print(f"Skipping {jsonl_path}: not in the cache yet") + return [] + if not silent: + print(f"Loading {jsonl_path} from cache...") + return committed # Try to load from cache first if cache_manager is not None: # Use filtered loading if date parameters are provided @@ -1337,6 +1353,7 @@ def load_directory_transcripts( silent: bool = False, persist_sidecar: bool = False, sidecar_required: bool = False, + snapshot: bool = False, ) -> tuple[list[TranscriptEntry], SessionTree]: """Load all JSONL transcript files from a directory and combine them. @@ -1352,6 +1369,7 @@ def load_directory_transcripts( ``sidecar_required`` makes a failed sidecar write raise on contention or drop the sidecar otherwise, instead of only being logged: the cache refresh passes it, see ``_persist_session_sidecar``. + ``snapshot`` is the render's load: see ``load_transcript``. """ all_messages: list[TranscriptEntry] = [] @@ -1369,7 +1387,12 @@ def load_directory_transcripts( with load_batch: for jsonl_file in jsonl_files: messages = load_transcript( - jsonl_file, cache_manager, from_date, to_date, silent + jsonl_file, + cache_manager, + from_date, + to_date, + silent, + snapshot=snapshot, ) all_messages.extend(messages) @@ -1595,6 +1618,7 @@ def _load_stale_session_transcripts( trunk_files, silent, entry_store=entry_store, + snapshot=True, ) @@ -1651,6 +1675,7 @@ def _load_sessions_partial( reelect_uuids: Optional[set[str]] = None, native_junction_uuids: Optional[set[str]] = None, entry_store: "Optional[ParsedEntryStore]" = None, + snapshot: bool = False, ) -> tuple[list[TranscriptEntry], SessionTree]: """Load the given trunk files into a faithful partial (entries, tree). @@ -1694,6 +1719,7 @@ def _load_sessions_partial( None, silent, entry_store=entry_store, + snapshot=snapshot, ) ) @@ -3411,7 +3437,7 @@ def _stream_paginated_conversion( if {f.name for f in trunk_files} != needed_files: return None page_entries, page_tree = _load_sessions_partial( - input_path, cache_manager, sidecar, trunk_files, silent + input_path, cache_manager, sidecar, trunk_files, silent, snapshot=True ) page_entries = deduplicate_messages(page_entries) @@ -4046,6 +4072,7 @@ def convert_jsonl_to( # A full unfiltered load is the moment the cross-session # sidecar can be (re)derived for the session-scoped path. persist_sidecar=(from_date is None and to_date is None), + snapshot=True, ) # Get working directories from cache @@ -5449,8 +5476,10 @@ def generate_single_session_file( # Ensure fresh cache ensure_fresh_cache(input_path, cache_manager, silent=True) - # Load messages from JSONL files - messages, _session_tree = load_directory_transcripts(input_path, cache_manager) + # Render what the refresh committed (see ``load_transcript``). + messages, _session_tree = load_directory_transcripts( + input_path, cache_manager, snapshot=True + ) # Collect all known session IDs: from loaded messages + cache metadata all_session_ids: set[str] = { diff --git a/claude_code_log/tui.py b/claude_code_log/tui.py index f812711a..7e7b54f3 100644 --- a/claude_code_log/tui.py +++ b/claude_code_log/tui.py @@ -1856,9 +1856,12 @@ def _ensure_session_file( # Load from cache for archived sessions messages = self.cache_manager.load_session_entries(session_id) else: - # Load from JSONL files for current sessions + # Refresh first, then render what the refresh committed + # (see ``load_transcript``): the cache may date from when + # the TUI started, and a session is still being written. + ensure_fresh_cache(self.project_path, self.cache_manager, silent=True) messages, _tree = load_directory_transcripts( - self.project_path, self.cache_manager, silent=True + self.project_path, self.cache_manager, silent=True, snapshot=True ) if not messages: return None diff --git a/dev-docs/application_model.md b/dev-docs/application_model.md index d4f89e75..7120c651 100644 --- a/dev-docs/application_model.md +++ b/dev-docs/application_model.md @@ -281,6 +281,10 @@ HTML exports honour `--theme` / `$CLAUDE_CODE_LOG_THEME` (§ 2.1): `html_theme` (not `theme`, which is Textual's own UI theme), and `_ensure_session_file` renders with it — regenerating a session page left in the other theme, under the same `session-{id}.html` name. +Before it renders a live session, `_ensure_session_file` refreshes the +cache (`ensure_fresh_cache`), then renders what that refresh committed +(§ 2.3): the cache the TUI read at start may predate the session's +newest lines. The "archive" action is interesting: it moves a session's source JSONL out of `~/.claude/projects/` while keeping the cache row intact. The @@ -379,6 +383,33 @@ project-wide and boolean: two processes refreshing the same project at once can still clear each other's, as they could already overwrite each other's session rows. +**A run renders what its refresh committed.** Only the refresh writes +entry rows; the render reads them and writes none. A session file can +grow between the two, since sessions are written to while a run converts +them. The render used to find that file's rows stale, parse it again and +write its rows itself, stamped with the file's new size and mtime, while +the session rows and aggregates kept the refresh's counts. The run's +pages then showed lines its session rows did not count, and the cache +called the file current, so only `refresh_pending` (above) made the next +run look at it again. Now every render-side load passes `snapshot=True` +to `load_transcript`, through `load_directory_transcripts` and +`_load_sessions_partial`: Phase 1b (§ 2.12), the streaming page loads +(§ 2.13), Phase 2, `generate_single_session_file` and the TUI's export. +In that mode the loader serves `CacheManager.load_snapshot_entries`, +the rows the cache holds at the current content version whatever the +file's stamp says now, or a parsed-entry store hit (§ 2.16), which is +verified against the file and so describes the same rows. It never +parses the file and never writes the cache. A file with no rows yet +(created after the refresh) is skipped. Lines that land after the +refresh are the next refresh's: that run finds the file modified, as +nothing restamped it. `--no-cache` and single-file mode have no rows and +parse as before. The hierarchy's no-cache-data fallback and Codex's +wholesale render load and write their rows and session rows from one +load, so they keep doing both. The TUI refreshes right before it exports +a session, because the cache it would otherwise render may date from its +start. `test/test_render_snapshot.py` grows a file right after the +refresh, on each of these paths. + **Corruption is recovered by discarding, not repairing.** A damaged cache file is unusable and, left alone, permanently so: `apply_migration` records only a migration that completed, so a `CREATE INDEX` that hits a @@ -1463,6 +1494,12 @@ states nobody sees; without the cap a long unbroken stream would never surface. `tick()` (poll once) is split from `run()` (wait) and the clock is injectable, so tests drive ticks by hand against a fake clock. +A tick stamps the files before it calls the conversion, and the +conversion renders what its own refresh committed (§ 2.3). So a line +written while a tick renders is in none of that tick's output, and it is +a change for the next tick, which refreshes and shows it: one tick late, +never lost. + **What makes a tick cheap** is §2.12's session-scoped path, which is reachable here only because `ensure_fresh_cache_detailed` reports *how* it refreshed (`CacheRefresh.NONE`/`INCREMENTAL`/`FULL`). Phase 1b's @@ -1631,7 +1668,9 @@ reason it is a parameter rather than a memo: `(size, mtime_ns)` against the stamp captured *before* the parse. A stamp taken before the parse can only be older than the entries describe, so a file that grew mid-parse declines to the cache rather - than serving a list its stamp misdescribes. The stamp also carries the + than serving a list its stamp misdescribes. A file that grew after the + refresh declines too, and the render then reads the rows the refresh + committed (§ 2.3), not the file. The stamp also carries the file's **sub-agent fingerprint** (`cache.subagents_fingerprint`, which covers the `agent-*.jsonl` transcripts as well as the sidecars, scoped for an agent file as its cache row is — `PendingStamp` snapshots before diff --git a/test/test_render_snapshot.py b/test/test_render_snapshot.py index 1dff5dd9..fc927068 100644 --- a/test/test_render_snapshot.py +++ b/test/test_render_snapshot.py @@ -20,6 +20,8 @@ from claude_code_log import converter from claude_code_log.cache import CacheManager, get_library_version +from claude_code_log.entry_store import ParsedEntryStore +from claude_code_log.watch import WatchEngine from test.test_date_filtered_runs import ( SESSIONS, @@ -27,6 +29,7 @@ _append, _build_project, _convert, + _second_day, _turn, ) @@ -47,57 +50,246 @@ def _session_row(project: Path, sid: str) -> Tuple[Any, ...]: ) -def _grow_after_the_refresh(monkeypatch: pytest.MonkeyPatch, path: Path) -> list[Path]: - """Append a turn to ``path`` once, right after the run's cache refresh.""" - refresh = converter.ensure_fresh_cache_detailed - grown: list[Path] = [] +class _GrowAfterTheRefresh: + """Append a late turn to ``path`` once, right after a run's cache + refresh, and record every entry-row write that follows it.""" - def refresh_then_grow(*args: Any, **kwargs: Any) -> Any: - result = refresh(*args, **kwargs) - if not grown: - _append(path, _turn(path.stem, 3, 3, "LATE-TURN")) - grown.append(path) + def __init__( + self, + monkeypatch: pytest.MonkeyPatch, + path: Path, + text: str = "LATE-TURN", + module: Any = converter, + refresh_name: str = "ensure_fresh_cache_detailed", + ) -> None: + self.path = path + self.grown = False + self.writes_after: list[str] = [] + refresh = getattr(module, refresh_name) + + def refresh_then_grow(*args: Any, **kwargs: Any) -> Any: + result = refresh(*args, **kwargs) + if not self.grown: + _append(path, _turn(path.stem, 3, 3, text)) + self.grown = True + return result + + monkeypatch.setattr(module, refresh_name, refresh_then_grow) + for name in ("save_cached_entries", "extend_cached_entries"): + monkeypatch.setattr(CacheManager, name, self._spy(name)) + + def _spy(self, name: str) -> Any: + write = getattr(CacheManager, name) + + def spied( + manager: CacheManager, jsonl_path: Path, *args: Any, **kwargs: Any + ) -> Any: + if self.grown: + self.writes_after.append(f"{name} {jsonl_path.name}") + return write(manager, jsonl_path, *args, **kwargs) + + return spied + + +# The paginated shape once more, with streaming forced, so the page loads +# of the streaming path are the render's loads. +STREAMED = "html-paginated-streamed" + + +def _shape(shape: str) -> str: + return "html-paginated" if shape == STREAMED else shape + + +def _counted(function: Any, calls: list[int]) -> Any: + """``function``, recording each call that did not decline (None).""" + + def counted(*args: Any, **kwargs: Any) -> Any: + result = function(*args, **kwargs) + if result is not None: + calls.append(1) return result - monkeypatch.setattr(converter, "ensure_fresh_cache_detailed", refresh_then_grow) - return grown + return counted + + +def _page(project: Path, shape: str) -> str: + ext = SHAPES[_shape(shape)]["output_format"] + return (project / f"session-{GROWN}.{ext}").read_text(encoding="utf-8") + + +def _cold_row( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, project: Path, shape: str +) -> Tuple[Any, ...]: + """The session row a cache built cold from the files gives.""" + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "control.db")) + _convert(project, _shape(shape)) + row = _session_row(project, GROWN) + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) + return row @pytest.mark.parametrize("warm", [False, True], ids=["cold", "warm"]) -@pytest.mark.parametrize("shape", ["html-combined", "md-sessions", "html-paginated"]) +@pytest.mark.parametrize( + "shape", ["html-combined", "md-sessions", "html-paginated", STREAMED] +) def test_a_file_grown_between_refresh_and_render_is_refreshed_next_run( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, shape: str, warm: bool ) -> None: project = _build_project(tmp_path) monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) + if shape == STREAMED: + monkeypatch.setenv("CLAUDE_CODE_LOG_STREAMING", "1") + streamed: list[int] = [] + monkeypatch.setattr( + converter, + "_stream_paginated_conversion", + _counted(converter._stream_paginated_conversion, streamed), + ) + shape_run = _shape(shape) jsonl = project / f"{GROWN}.jsonl" if warm: # The daily shape: a converted project whose session moved on, so # the refresh is the incremental one over a populated cache. - _convert(project, shape) + _convert(project, shape_run) _append(jsonl, _turn(GROWN, 2, 2, "WARM-TURN")) with monkeypatch.context() as m: - grown = _grow_after_the_refresh(m, jsonl) - _convert(project, shape) - assert grown == [jsonl], "the file never grew between refresh and render" + grow = _GrowAfterTheRefresh(m, jsonl) + _convert(project, shape_run) + assert grow.grown, "the file never grew between refresh and render" # The run's outputs agree: its session page shows the late turn exactly # when its session row counts it. - ext = SHAPES[shape]["output_format"] - page = (project / f"session-{GROWN}.{ext}").read_text(encoding="utf-8") + page = _page(project, shape) assert ("LATE-TURN" in page) == (_session_row(project, GROWN)[1] == LATE_STAMP), ( f"the page {'shows' if 'LATE-TURN' in page else 'lacks'} the late turn," f" the session row says {_session_row(project, GROWN)}" ) + # And they agree on the refresh's moment: the render read what the + # refresh committed, wrote no entry rows, and left the late turn out. + assert grow.writes_after == [] + assert _second_day(GROWN) in page, "the render lost what the refresh committed" + if warm: + assert "WARM-TURN" in page, "the render lost what the refresh committed" + assert "LATE-TURN" not in page + if shape == STREAMED: + assert streamed, "the run did not stream" # Nothing changes now: this run must still bring the row up to the file. - _convert(project, shape) + _convert(project, shape_run) + assert "LATE-TURN" in _page(project, shape) + assert _session_row(project, GROWN) == _cold_row( + tmp_path, monkeypatch, project, shape + ) - # Control: a cache built cold from the same files. - monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "control.db")) + +def test_a_line_landing_mid_render_shows_on_the_next_watch_tick( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Watch mode runs a conversion per tick, holding one entry store across + them. A line appended while a tick renders is left out of that tick's + pages and is not lost: the engine stamped the files before converting, + so the append is a change, and the next tick shows it.""" + project = _build_project(tmp_path) + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) + jsonl = project / f"{GROWN}.jsonl" + shape = "html-combined" _convert(project, shape) - expected = _session_row(project, GROWN) + store = ParsedEntryStore() + now = [0.0] + + def convert(_paths: set[Path]) -> None: + converter.convert_jsonl_to("html", project, silent=True, entry_store=store) + + engine = WatchEngine( + [project.parent], + convert, + quiet_period=1.0, + max_latency=5.0, + clock=lambda: now[0], + ) + engine.prime() + + def tick_after(seconds: float) -> Any: + now[0] += seconds + return engine.tick() + + _append(jsonl, _turn(GROWN, 2, 2, "WARM-TURN")) + assert tick_after(0) is None # seen, not yet quiet + with monkeypatch.context() as m: + grow = _GrowAfterTheRefresh(m, jsonl) + assert tick_after(1.0) == {jsonl} + assert grow.grown and grow.writes_after == [] + page = _page(project, shape) + assert "WARM-TURN" in page + assert "LATE-TURN" not in page + + assert tick_after(0) is None # the late turn, seen + assert tick_after(1.0) == {jsonl} + assert "LATE-TURN" in _page(project, shape) + assert engine.stats.conversions == 2 + assert _session_row(project, GROWN) == _cold_row( + tmp_path, monkeypatch, project, shape + ) + + +@pytest.mark.tui +def test_a_tui_export_renders_lines_written_since_the_tui_started( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The TUI refreshes the cache before it exports a session, since it + renders what the cache holds and that may date from its start.""" + from claude_code_log.tui import SessionBrowser + + project = _build_project(tmp_path) + monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) + _convert(project, "html-combined") + app = SessionBrowser(project) + + _append(project / f"{GROWN}.jsonl", _turn(GROWN, 2, 3, "SINCE-THE-TUI-STARTED")) + session_file = app._ensure_session_file(GROWN, "html", force=True) + + assert session_file is not None + assert "SINCE-THE-TUI-STARTED" in session_file.read_text(encoding="utf-8") + assert _session_row(project, GROWN) == _cold_row( + tmp_path, monkeypatch, project, "html-combined" + ) + + # And it renders what that refresh committed, as a run does. + from claude_code_log import tui + + with monkeypatch.context() as m: + grow = _GrowAfterTheRefresh( + m, project / f"{GROWN}.jsonl", module=tui, refresh_name="ensure_fresh_cache" + ) + app._ensure_session_file(GROWN, "html", force=True) + assert grow.grown and grow.writes_after == [] + page = session_file.read_text(encoding="utf-8") + assert "SINCE-THE-TUI-STARTED" in page + assert "LATE-TURN" not in page + + +def test_a_single_session_export_renders_what_its_refresh_committed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project = _build_project(tmp_path) monkeypatch.setenv("CLAUDE_CODE_LOG_CACHE_PATH", str(tmp_path / "cache.db")) - assert _session_row(project, GROWN) == expected + _convert(project, "html-combined") + _append(project / f"{GROWN}.jsonl", _turn(GROWN, 2, 2, "WARM-TURN")) + + with monkeypatch.context() as m: + grow = _GrowAfterTheRefresh( + m, project / f"{GROWN}.jsonl", refresh_name="ensure_fresh_cache" + ) + output = converter.generate_single_session_file("html", project, GROWN) + assert grow.grown and grow.writes_after == [] + page = output.read_text(encoding="utf-8") + assert "WARM-TURN" in page + assert "LATE-TURN" not in page + + output = converter.generate_single_session_file("html", project, GROWN) + assert "LATE-TURN" in output.read_text(encoding="utf-8") + assert _session_row(project, GROWN) == _cold_row( + tmp_path, monkeypatch, project, "html-combined" + ) From 8d21a3dc9a932d3f9aca6f833e07f40e9ebd6e1e Mon Sep 17 00:00:00 2001 From: Christian Boos Date: Sat, 7 Nov 2026 16:13:54 +0100 Subject: [PATCH 3/3] Keep the surrogate export test off the TUI's new cache refresh The test mocks `get_cached_project_data` and `load_directory_transcripts` so that the TUI's export never loads a transcript. The export now refreshes the cache first, and that refresh read the mocked project data, failed, and the export returned None. The test mocks the refresh as well: what it checks is the scrubbing of the written page. Co-Authored-By: Claude Opus 5.5 --- test/test_tui_surrogate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_tui_surrogate.py b/test/test_tui_surrogate.py index 68570054..7cb28dad 100644 --- a/test/test_tui_surrogate.py +++ b/test/test_tui_surrogate.py @@ -90,13 +90,16 @@ def test_session_export_scrubs_lone_surrogate(self, tmp_path: Path, surrogate: s # Mock the renderer's generate_session to inject surrogate-bearing # content directly. Also short-circuit `is_outdated` so we always - # write, and `load_directory_transcripts` to skip JSONL parsing. + # write, and `load_directory_transcripts` to skip JSONL parsing, + # and with it the cache refresh before it, which would read the + # mocked project data below. mock_renderer = MagicMock() mock_renderer.is_outdated.return_value = True mock_renderer.generate_session.return_value = surrogate_html with ( patch("claude_code_log.tui.get_renderer", return_value=mock_renderer), + patch("claude_code_log.tui.ensure_fresh_cache"), patch( "claude_code_log.tui.load_directory_transcripts", return_value=([{"placeholder": True}], None),