fix: allow negative TOC offset scan and page-scaled TOC max_tokens

apps/worker/app/services/document_agent/calibration/phase1.py

@@ -5,10 +5,15 @@
 and scans forward until one is confirmed; that single confirmation fixes the
 regime's candidate offset. Phase-2 owns tail verification and bulk anchoring.
 
-A probe scans from ``max(toc_range end + 1, printed)``: a printed label never
-resolves to a physical page before itself, so the offset a scan can yield is
-structurally non-negative. Scanning below the printed page would let a section
-divider that repeats the heading confirm ahead of the numbered body page.
+A probe scans from ``toc_range end + 1`` (body after the TOC). That can yield a
+negative ``physical - printed`` offset (e.g. journal reprints whose printed
+labels exceed the PDF length). Phase-2 already accepts negative offsets and
+prunes ``printed + offset`` outside ``1..page_count``.
+
+To restore the Sydney anti-false-positive floor (non-negative offset), switch
+``start_page`` back to ``max(region_scan_start, probe.printed)`` below and
+invert ``test_scan_starts_after_toc_so_negative_offset_is_allowed`` to expect
+scan start ``= printed`` and offset ``0``.
 """
 
 from __future__ import annotations
@@ -150,7 +155,10 @@ def run_calibration_phase1(
             scan = scan_title_forward(
                 ctx=ctx,
                 title=probe.title,
-                start_page=max(region_scan_start, probe.printed),
+                # Scan from after TOC. Optional floor (disabled): force
+                # non-negative offset / skip early divider hits — restore with:
+                #   start_page=max(region_scan_start, probe.printed),
+                start_page=region_scan_start,
                 page_count=resolved_page_count,
             )
             scans.append(scan)

apps/worker/app/services/document_agent/tools/extract_toc_with_boundaries.py

@@ -19,7 +19,6 @@
 )
 from app.services.document_agent.registry import register_tool
 from app.services.document_agent.tools.vlm_toc_extractor import (
-    TOC_VLM_MAX_TOKENS,
     BatchPageResult,
     vlm_entries_to_toc_hierarchies,
 )
@@ -35,6 +34,8 @@
 TOC_VLM_CONCURRENCY = 10
 MAX_BOUNDARY_ROUNDS = 6
 MAX_TOC_PAGES = BOUNDARY_STEP_PAGES * MAX_BOUNDARY_ROUNDS  # 30
+# Confirm only returns is_toc_start + brief reason per page (≤ BOUNDARY_STEP_PAGES).
+TOC_ANCHOR_CONFIRM_MAX_TOKENS = 512
 
 _CONFIRM_PROMPT = (
     "You are a document structure analysis expert. "
@@ -183,7 +184,7 @@ def _confirm_anchor_chunk(
             messages=messages,
             model=resolved,
             temperature=0.1,
-            max_tokens=TOC_VLM_MAX_TOKENS,
+            max_tokens=TOC_ANCHOR_CONFIRM_MAX_TOKENS,
             response_format={"type": "json_object"},
             usage_task="document_agent.toc_anchor_confirm",
         )

apps/worker/app/services/document_agent/tools/vlm_toc_extractor.py

@@ -8,8 +8,13 @@
 from dataclasses import dataclass
 from typing import Any, cast
 
-# Shared completion budget for TOC VLM calls (confirm batches + extract batches).
-TOC_VLM_MAX_TOKENS = 8192
+# Completion budget for TOC VLM calls: known pages in the call × per-page cap.
+TOC_VLM_MAX_TOKENS_PER_PAGE = 3000
+
+
+def toc_vlm_max_tokens(page_count: int) -> int:
+    """Return ``max_tokens`` for a TOC VLM call covering ``page_count`` pages."""
+    return max(1, int(page_count)) * TOC_VLM_MAX_TOKENS_PER_PAGE
 
 
 # ---------------------------------------------------------------------------
@@ -199,7 +204,7 @@ def vlm_extract_toc_batch(
         messages=cast(Any, [{"role": "user", "content": content_parts}]),
         model=model,
         temperature=0.1,
-        max_tokens=TOC_VLM_MAX_TOKENS,
+        max_tokens=toc_vlm_max_tokens(len(page_pngs)),
         response_format={"type": "json_object"},
         usage_task="document_agent.vlm_toc_batch",
     )

apps/worker/tests/contract/test_calibration_phase1_contract.py

@@ -96,8 +96,8 @@ def test_offset_is_found_page_minus_printed(patch_scan) -> None:
 
     assert result.status == "ok"
     assert [(r.kind, r.offset) for r in result.regimes] == [("decimal", 5)]
-    # toc_range=[1, 3], printed=10 → scan starts at the printed page.
-    assert fake.calls == [("Chapter 1", 10)]
+    # toc_range=[1, 3] → scan starts at toc_end + 1 (not printed).
+    assert fake.calls == [("Chapter 1", 4)]
 
 
 def test_first_hit_stops_the_regime(patch_scan) -> None:
@@ -111,7 +111,7 @@ def test_first_hit_stops_the_regime(patch_scan) -> None:
 
     run_calibration_phase1(ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60)
 
-    assert fake.calls == [("Chapter 1", 10)]
+    assert fake.calls == [("Chapter 1", 4)]
 
 
 def test_second_probe_runs_when_the_first_misses(patch_scan) -> None:
@@ -128,7 +128,7 @@ def test_second_probe_runs_when_the_first_misses(patch_scan) -> None:
         ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60
     )
 
-    assert fake.calls == [("Chapter 1", 10), ("Chapter 2", 20)]
+    assert fake.calls == [("Chapter 1", 4), ("Chapter 2", 4)]
     assert [r.offset for r in result.regimes] == [5]
 
 
@@ -146,7 +146,7 @@ def test_probes_use_distinct_printed_pages(patch_scan) -> None:
         ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60
     )
 
-    assert fake.calls == [("Chapter 1", 10), ("Chapter 2", 20)]
+    assert fake.calls == [("Chapter 1", 4), ("Chapter 2", 4)]
     assert [r.offset for r in result.regimes] == [5]
 
 
@@ -164,7 +164,7 @@ def test_probe_prefers_leaf_within_a_printed_page(patch_scan) -> None:
         ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60
     )
 
-    assert fake.calls == [("A1 Purpose", 10)]
+    assert fake.calls == [("A1 Purpose", 4)]
     assert [r.offset for r in result.regimes] == [5]
 
 
@@ -205,14 +205,18 @@ def test_roman_and_decimal_regimes_calibrate_independently(patch_scan) -> None:
         ("roman", 2),
         ("decimal", 5),
     }
-    # printed=2 sits inside the TOC range → floor at toc end + 1; printed=10 wins.
-    assert fake.calls == [("Preface", 4), ("Chapter 1", 10)]
+    # Both regimes scan from toc_end + 1 (printed floor disabled).
+    assert fake.calls == [("Preface", 4), ("Chapter 1", 4)]
 
 
-def test_scan_floor_is_the_printed_page_so_offset_is_never_negative(
+def test_scan_starts_after_toc_so_negative_offset_is_allowed(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """A divider repeating the heading cannot confirm ahead of the printed page."""
+    """Confirming on the first post-TOC page can yield physical < printed.
+
+    Restore ``max(region_scan_start, probe.printed)`` in phase1 to bring back
+    ``test_scan_floor_is_the_printed_page_so_offset_is_never_negative`` behavior.
+    """
 
     class _ConfirmFirstPage:
         def __init__(self) -> None:
@@ -246,8 +250,8 @@ def __call__(
         ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60
     )
 
-    assert fake.calls == [("Chapter 1", 10)]
-    assert [r.offset for r in result.regimes] == [0]
+    assert fake.calls == [("Chapter 1", 4)]
+    assert [r.offset for r in result.regimes] == [-6]
 
 
 def test_confirmed_anchor_is_reported_as_a_sample(patch_scan) -> None:
@@ -275,7 +279,7 @@ def test_entries_without_a_parseable_printed_page_are_skipped(patch_scan) -> Non
 
     run_calibration_phase1(ctx=_ctx(), toc_hierarchies=hierarchies, page_count=60)
 
-    assert fake.calls == [("Chapter 1", 10)]
+    assert fake.calls == [("Chapter 1", 4)]
 
 
 def test_empty_toc_fails_without_scanning(patch_scan) -> None: