Report omitted local notes in overlap searches

docs/research/coordination.md

@@ -76,6 +76,33 @@ unstructured discussions are inactive. Multiple/malformed metadata blocks are
 reported, not silently treated as valid claims. Task IDs should match the
 catalogue where possible; a genuinely different scope can use a descriptive ID.
 
+## Local overlap coverage
+
+`find_overlap.py` searches working-tree `research/*/README.md`,
+`research/*/TASK.json`, and `docs/research/pr-ledger.md`. A sparse checkout can
+omit most of these notes even when both remote snapshots are complete.
+`coverage.local` records the searched count (`records`), the pinned `head`,
+and the count of regular note files tracked there (`head_records`).
+In `--json` output, `unsearched_head_paths` lists every tracked note path absent
+from the searched corpus. Text output shows `unsearched_head_paths_count` to
+avoid printing thousands of omitted paths. Omissions produce a warning even
+with no matches or `--limit 1`.
+They can arise from sparse checkout, ordinary deletion, or a skipped symlink;
+the report does not infer the cause. Materialize relevant missing notes and
+rerun the search before relying on their coverage.
+
+`complete_head_inventory` is true only when every in-scope regular HEAD note
+path was searched. It is false for known omissions and null when Git or the
+HEAD inventory is unavailable, with an explicit warning. A complete empty
+inventory has count zero and completeness true. This compares paths, not
+content identities: local edits and untracked notes remain searchable, so
+`records` can exceed `head_records`. Nested vendor notes, arbitrary package
+files, and historical versions remain outside this local search. Git tree
+metadata is inspected without reading omitted blobs or expanding the checkout;
+local coverage is computed on each invocation and is not stored in the remote
+cache. Regression fixtures cover full, non-cone sparse, and cone sparse-index
+checkouts in `tests/test_research_overlap_local.py`.
+
 ## Changed-path overlap snapshots
 
 A `find_overlap.py --path PATTERN` report binds each file inventory to both

scripts/find_overlap.py

@@ -323,6 +323,46 @@ def local_records(root):
     return records
 
 
+def local_coverage(root, records):
+    """Compare searched working-tree notes with the regular note paths at HEAD."""
+    coverage = {"head": None, "records": len(records), "head_records": None,
+                "complete_head_inventory": None, "unsearched_head_paths": None}
+    try:
+        def git(*args):
+            return subprocess.run(
+                ["git", "--no-replace-objects", *args], cwd=root, check=True,
+                capture_output=True, encoding="utf-8", errors="surrogateescape", timeout=15,
+                env={**os.environ, "GIT_NO_LAZY_FETCH": "1"},
+            ).stdout
+
+        # Pin the tree once; only metadata is read, never omitted note blobs.
+        coverage["head"] = git("rev-parse", "--verify", "HEAD^{commit}").strip()
+        tree = git("ls-tree", "-r", "-z", "--full-tree", coverage["head"], "--",
+                   "research", "docs/research/pr-ledger.md")
+        expected = set()
+        for entry in tree.split("\0"):
+            if not entry:
+                continue
+            metadata, path = entry.split("\t", 1)
+            mode, kind, _ = metadata.split()
+            parts = PurePosixPath(path).parts
+            in_scope = (path == "docs/research/pr-ledger.md"
+                        or (len(parts) == 3 and parts[0] == "research"
+                            and parts[2] in ("README.md", "TASK.json")))
+            if in_scope and kind == "blob" and mode in ("100644", "100755"):
+                expected.add(path)
+    except (OSError, subprocess.SubprocessError, ValueError) as exc:
+        return coverage, ["local: HEAD note inventory unavailable; local coverage is unknown. "
+                          + error_summary(exc)]
+    missing = sorted(expected - {record["path"] for record in records})
+    coverage.update(head_records=len(expected), complete_head_inventory=not missing,
+                    unsearched_head_paths=missing)
+    warnings = ([f"local: {len(missing)} HEAD note paths were not searched; "
+                 "inspect coverage.local.unsearched_head_paths in --json output and materialize relevant notes "
+                 "before treating a lack of matches as evidence."] if missing else [])
+    return coverage, warnings
+
+
 def match_position(text, term, pattern):
     if pattern is None:
         return text.find(term)
@@ -403,10 +443,12 @@ def main():
         ap.error("provide terms, --task or --path")
     patterns = [p.strip() for p in args.path if p.strip()]
     local = local_records(ROOT)
+    local_status, local_warnings = local_coverage(ROOT, local)
     snapshot, warnings = remote_snapshot(args.cache, args.offline, args.refresh, args.max_age,
                                          need_files=bool(patterns), include_closed=args.include_closed)
+    warnings = local_warnings + warnings
     records = list(local) + pull_corpus(snapshot, args.include_closed)
-    coverage = {}
+    coverage = {"local": local_status}
     corpus_names = ("open_prs", "discussions", "closed_prs") if args.include_closed else ("open_prs", "discussions")
     for name in corpus_names:
         source = snapshot["sources"].get(name)
@@ -461,7 +503,9 @@ def main():
                           "State changes are deduplicated by latest PR update; snapshots are not atomic. "
                           if args.include_closed else
                           "Remote PR scope is open PRs including drafts. Closed work is searched through local READMEs, TASK metadata and ledger only. ")
-                         + "No code diffs or arbitrary package files are searched. Recheck relevant heads before submission."),
+                         + "Local coverage compares searched working-tree notes with regular note paths at coverage.local.head; "
+                         "it does not certify HEAD contents or historical coverage. "
+                         "No code diffs or arbitrary package files are searched. Recheck relevant heads before submission."),
               "matches_total": len(matches), "matches": matches[:args.limit]}
     if patterns:
         report["path_patterns"] = patterns
@@ -482,7 +526,12 @@ def main():
         if hasattr(sys.stdout, "reconfigure"):
             sys.stdout.reconfigure(errors="backslashreplace")
         print(report["status"])
-        print("Coverage: " + json.dumps(coverage))
+        # A sparse checkout can omit thousands of notes. Keep the complete
+        # path list in JSON; the text report surfaces the count and warning.
+        local_summary = {k: v for k, v in local_status.items() if k != "unsearched_head_paths"}
+        omitted = local_status.get("unsearched_head_paths")
+        local_summary["unsearched_head_paths_count"] = None if omitted is None else len(omitted)
+        print("Coverage: " + json.dumps({**coverage, "local": local_summary}))
         if reference_terms:
             print("Task references (identifier matching): " + ", ".join(reference_terms))
         print(report["limits"])

tests/test_research_discussion_batching.py

@@ -131,6 +131,7 @@ def test_cli_finds_a_reply_beyond_all_page_boundaries(self):
             output = io.StringIO()
             argv = ["find_overlap.py", "D0C206R206", "--json", "--cache", str(cache)]
             with (patch.object(overlap, "ROOT", root), patch.object(sys, "argv", argv),
+                  patch.object(overlap, "local_coverage", return_value=({}, [])),
                   patch.object(overlap, "graphql", side_effect=board.api),
                   patch.object(overlap, "gh_json", return_value=[[]]), redirect_stdout(output)):
                 overlap.main()

tests/test_research_overlap_closed.py

@@ -38,6 +38,7 @@ class ClosedOverlapTests(unittest.TestCase):
     def run_report(self, cache, argv=(), closed_pages=None, open_pages=None, seen=None):
         output = io.StringIO()
         with (patch.object(overlap, 'ROOT', cache.parent / 'empty-repo'),
+              patch.object(overlap, 'local_coverage', return_value=({}, [])),
               patch.object(sys, 'argv', ['find_overlap.py', 'needle', '--json', '--cache', str(cache), *argv]),
               patch.object(overlap.subprocess, 'run', side_effect=cli(open_pages or [[]], closed_pages or [[]], seen if seen is not None else [])),
               patch.object(overlap, 'fetch_discussions', return_value=[]),

tests/test_research_overlap_local.py

@@ -0,0 +1,165 @@
+"""Local overlap coverage must expose notes omitted from a Git checkout."""
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+@unittest.skipUnless(shutil.which("git"), "Git is needed for checkout fixtures")
+class LocalOverlapCoverageTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        self.root = Path(temporary.name) / "repo"
+        self.root.mkdir()
+        self.cache = Path(temporary.name) / "cache.json"
+        sources = {name: {"fetched_at": datetime.now(timezone.utc).isoformat(),
+                          "complete_pagination": True, "records": []}
+                   for name in ("open_prs", "discussions")}
+        self.cache.write_text(json.dumps({"schema_version": 1,
+                                         "repository": "CrocSwap/integer-mult-bounds",
+                                         "sources": sources}), encoding="utf-8")
+        self.omitted = "research/omitted κ\npackage"
+        for path, body in {
+            "scripts/find_overlap.py": (ROOT / "scripts/find_overlap.py").read_text(encoding="utf-8"),
+            "research/kept/README.md": "needle local note",
+            "research/kept/TASK.json": '{"scope": "needle local task"}',
+            f"{self.omitted}/README.md": "only-omitted needle",
+            f"{self.omitted}/TASK.json": '{"scope": "only-omitted needle"}',
+            "docs/research/pr-ledger.md": "ledger needle",
+            "research/kept/vendor/README.md": "out-of-scope nested note",
+            "research/kept/NOTES.md": "out-of-scope arbitrary note",
+        }.items():
+            self.write(path, body)
+        self.git("init", "-q")
+        self.git("add", ".")
+        self.git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid",
+                 "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false",
+                 "commit", "-qm", "overlap fixture")
+        self.head = self.git("rev-parse", "HEAD").strip()
+
+    def write(self, name, body):
+        path = self.root / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(body, encoding="utf-8")
+
+    def git(self, *args):
+        return subprocess.run(["git", *args], cwd=self.root, check=True,
+                              capture_output=True, encoding="utf-8").stdout
+
+    def report(self, *args, text=False, env=None):
+        original = self.cache.read_bytes()
+        result = subprocess.run(
+            [sys.executable, "-I", "-B", str(self.root / "scripts/find_overlap.py"),
+             "--cache", str(self.cache), "--max-age", "3600",
+             *([] if text else ["--json"]), *args],
+            cwd=self.root, capture_output=True, encoding="utf-8", timeout=15, env=env)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.cache.read_bytes(), original)
+        return result.stdout if text else json.loads(result.stdout)
+
+    def sparse(self, cone=False):
+        if cone:
+            self.git("sparse-checkout", "init", "--cone", "--sparse-index")
+            self.git("sparse-checkout", "set", "scripts", "research/kept")
+        else:
+            self.git("sparse-checkout", "init", "--no-cone")
+            self.git("sparse-checkout", "set", "/scripts/", "/research/kept/")
+
+    def test_full_checkout_uses_working_edits_and_untracked_notes(self):
+        self.write("research/kept/README.md", "edited needle")
+        self.write("research/untracked/README.md", "new needle")
+        report = self.report("needle")
+        self.assertEqual(report["local_records"], 6)
+        self.assertEqual(report["coverage"]["local"], {
+            "head": self.head, "records": 6, "head_records": 5,
+            "complete_head_inventory": True, "unsearched_head_paths": []})
+        self.assertEqual(report["warnings"], [])
+        self.assertEqual({m["excerpt"] for m in report["matches"] if "README.md" in m["path"]},
+                         {"edited needle", "new needle", "only-omitted needle"})
+
+    def test_sparse_gaps_survive_no_hits_and_result_limits(self):
+        expected = sorted(["docs/research/pr-ledger.md", f"{self.omitted}/README.md",
+                           f"{self.omitted}/TASK.json"])
+        for cone in (False, True):
+            with self.subTest(cone=cone):
+                self.sparse(cone)
+                report = self.report("only-omitted", "--offline", "--limit", "1")
+                self.assertEqual(report["matches_total"], 0)
+                self.assertEqual(report["coverage"]["local"], {
+                    "head": self.head, "records": 2, "head_records": 5,
+                    "complete_head_inventory": False, "unsearched_head_paths": expected})
+                self.assertTrue(any("local: 3 HEAD note paths were not searched" in warning
+                                    for warning in report["warnings"]))
+                limited = self.report("needle", "--limit", "1")
+                self.assertEqual(limited["matches_total"], 2)
+                self.assertEqual(len(limited["matches"]), 1)
+                self.assertEqual(limited["coverage"]["local"], report["coverage"]["local"])
+                output = self.report("only-omitted", text=True)
+                self.assertIn('"complete_head_inventory": false', output)
+                self.assertIn('"unsearched_head_paths_count": 3', output)
+                self.assertNotIn('"unsearched_head_paths":', output)
+                self.assertIn("WARNING: local: 3 HEAD note paths were not searched", output)
+                self.assertFalse((self.root / self.omitted).exists())
+                self.git("sparse-checkout", "disable")
+                restored = self.report("only-omitted")
+                self.assertTrue(restored["coverage"]["local"]["complete_head_inventory"])
+                self.assertEqual(restored["matches_total"], 2)
+
+    def test_ordinary_deleted_note_is_also_unsearched(self):
+        (self.root / "research/kept/README.md").unlink()
+        report = self.report("needle")
+        self.assertEqual(report["coverage"]["local"]["unsearched_head_paths"],
+                         ["research/kept/README.md"])
+        self.assertFalse(report["coverage"]["local"]["complete_head_inventory"])
+
+    def test_skipped_symlink_cannot_count_as_a_searched_head_note(self):
+        note = self.root / "research/kept/README.md"
+        note.unlink()
+        try:
+            note.symlink_to("TASK.json")
+        except OSError:
+            self.skipTest("symlinks unavailable")
+        report = self.report("needle")
+        self.assertEqual(report["coverage"]["local"]["unsearched_head_paths"],
+                         ["research/kept/README.md"])
+        self.assertEqual(report["local_records"], 4)
+
+    def test_empty_head_note_inventory_is_complete(self):
+        self.git("rm", "-qr", "research", "docs")
+        self.git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid",
+                 "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false",
+                 "commit", "-qm", "no notes")
+        report = self.report("needle")
+        self.assertEqual(report["coverage"]["local"]["head_records"], 0)
+        self.assertTrue(report["coverage"]["local"]["complete_head_inventory"])
+        self.assertEqual(report["warnings"], [])
+
+    def test_missing_git_or_repository_keeps_search_but_marks_coverage_unknown(self):
+        for mode in ("missing-git", "not-a-repository", "unborn-head"):
+            with self.subTest(mode=mode):
+                if mode == "not-a-repository":
+                    shutil.rmtree(self.root / ".git")
+                elif mode == "unborn-head":
+                    self.git("init", "-q")
+                env = {**os.environ, "PATH": ""} if mode == "missing-git" else None
+                report = self.report("needle", env=env)
+                coverage = report["coverage"]["local"]
+                self.assertIsNone(coverage["complete_head_inventory"])
+                self.assertIsNone(coverage["head_records"])
+                self.assertIsNone(coverage["unsearched_head_paths"])
+                self.assertEqual(report["matches_total"], 5)
+                self.assertTrue(any(w.startswith("local: HEAD note inventory unavailable;")
+                                    for w in report["warnings"]))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_research_overlap_pagination.py

@@ -52,6 +52,7 @@ def test_cli_finds_later_page_draft_without_slurp(self):
             output = io.StringIO()
             argv = ["find_overlap.py", "dirty decoder", "--json", "--cache", str(cache)]
             with (patch.object(overlap, "ROOT", root), patch.object(sys, "argv", argv),
+                  patch.object(overlap, "local_coverage", return_value=({}, [])),
                   patch.object(overlap.subprocess, "run", side_effect=legacy_cli(pages)),
                   patch.object(overlap, "fetch_discussions", return_value=[]),
                   contextlib.redirect_stdout(output)):

tests/test_research_overlap_terms.py

@@ -44,7 +44,11 @@ def report(self, card, records, *args):
                          {"fetched_at": sources["open_prs"]["fetched_at"],
                           "complete_pagination": True, "records": len(records)})
         self.assertEqual(report["status"], "POSSIBLE_OVERLAP_NOT_A_NOVELTY_CERTIFICATE")
-        self.assertEqual(len(report["warnings"]), 2)
+        # This copied-script fixture is not a Git checkout; only local inventory
+        # coverage is unknown, independently of the two offline-source warnings.
+        self.assertIsNone(report["coverage"]["local"]["complete_head_inventory"])
+        self.assertEqual(sum(w.startswith("local:") for w in report["warnings"]), 1)
+        self.assertEqual(len(report["warnings"]), 3)
         return report
 
     def test_nested_evidence_does_not_hide_a_lead_below_the_default_limit(self):

tests/test_research_pr_file_pages.py

@@ -155,6 +155,7 @@ def test_cli_covers_missing_inventory_and_matching_head_control(self):
                                              'sources': sources}), encoding='utf-8')
                 output = io.StringIO()
                 with (patch.object(overlap, 'ROOT', Path(directory) / 'repo'),
+                      patch.object(overlap, 'local_coverage', return_value=({}, [])),
                       patch.object(sys, 'argv', ['find_overlap.py', '--path', 'new.py',
                                                 '--json', '--cache', str(cache)]),
                       contextlib.redirect_stdout(output),

tests/test_research_pr_inventory_coverage.py

@@ -84,7 +84,10 @@ def test_text_report_surfaces_unmatched_gaps_and_combined_match_flags(self):
     def test_complete_empty_inventory_is_not_reported_as_a_gap(self):
         report = json.loads(self.run_cli([self.row(376, [], False)], '--path', 'missing.py', '--json'))
         self.assertEqual(report['coverage']['pr_files']['truncated_prs'], [])
-        self.assertEqual(report['warnings'], [])
+        # PATH is empty in this CLI fixture, so Git inventory is unavailable.
+        self.assertIsNone(report['coverage']['local']['complete_head_inventory'])
+        self.assertEqual(len(report['warnings']), 1)
+        self.assertTrue(report['warnings'][0].startswith('local: HEAD note inventory unavailable;'))
         self.assertEqual(report['path_matches_total'], 0)