fix(vector): align MESA_Data evidence with V4 vector contract

pyproject.toml

@@ -59,4 +59,5 @@ ignore = ["BLE001", "B008", "DTZ007", "E501", "W293", "W291"]
 [tool.pytest.ini_options]
 markers = [
     "scale: Large-scale or heavy benchmark tests processing thousands of records",
+    "cross_repo: Optional compatibility test against a checked-out MESA source tree",
 ]

src/mesa_legal_data/publisher/chunker.py

@@ -1,7 +1,7 @@
 from typing import Any
 
 from mesa_legal_data.publisher.hashing import calculate_content_hash
-from mesa_legal_data.publisher.models import SourceChunk
+from mesa_legal_data.publisher.models import MESA_V4_EVIDENCE_SPAN_MAX_CHARS, SourceChunk
 
 
 def _split_oversized_text(
@@ -82,6 +82,12 @@ def plan_source_chunks(
     if not canonical_text or not canonical_text.strip():
         return []
 
+    # MESA's current V4 insert contract accepts at most 4,096 characters in
+    # ``evidence_span``.  Every source chunk becomes the complete evidence
+    # payload for its deterministic fallback assertion, so the semantic limit
+    # is stricter than the 32,768-character transport-content limit.
+    chunk_limit_chars = min(content_limit_chars, MESA_V4_EVIDENCE_SPAN_MAX_CHARS)
+
     # Auto-resolve char_start and char_end if content is provided
     search_cursor = 0
     for r in records:
@@ -105,7 +111,7 @@ def plan_source_chunks(
 
     if not article_records:
         # Generic text without article records (e.g. unsegmented or single decision)
-        splits = _split_oversized_text(canonical_text, 0, content_limit_chars)
+        splits = _split_oversized_text(canonical_text, 0, chunk_limit_chars)
         for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1):
             chunk_id = (
                 f"{version_id}:chunk:{ordinal_counter:04d}"
@@ -136,7 +142,7 @@ def plan_source_chunks(
     if first_art_start > 0:
         preamble_text = canonical_text[:first_art_start]
         if preamble_text.strip():
-            splits = _split_oversized_text(preamble_text, 0, content_limit_chars)
+            splits = _split_oversized_text(preamble_text, 0, chunk_limit_chars)
             for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1):
                 chunk_id = (
                     f"{version_id}:chunk:{ordinal_counter:04d}"
@@ -170,7 +176,7 @@ def plan_source_chunks(
         art_num = art.get("article_number", "")
         art_title = art.get("title") or (f"Madde {art_num}" if art_num else "Madde")
 
-        splits = _split_oversized_text(art_text, a_start, content_limit_chars)
+        splits = _split_oversized_text(art_text, a_start, chunk_limit_chars)
         for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1):
             chunk_id = (
                 f"{version_id}:chunk:{ordinal_counter:04d}"
@@ -205,7 +211,7 @@ def plan_source_chunks(
     if last_art_end < len(canonical_text):
         annex_text = canonical_text[last_art_end:]
         if annex_text.strip():
-            splits = _split_oversized_text(annex_text, last_art_end, content_limit_chars)
+            splits = _split_oversized_text(annex_text, last_art_end, chunk_limit_chars)
             for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1):
                 chunk_id = (
                     f"{version_id}:chunk:{ordinal_counter:04d}"

src/mesa_legal_data/publisher/client.py

@@ -6,6 +6,7 @@
 import httpx
 
 from mesa_legal_data.publisher.models import (
+    MESA_V4_EVIDENCE_SPAN_MAX_CHARS,
     MesaTargetSettings,
     MutationState,
     PreflightCheckItem,
@@ -391,6 +392,11 @@ def build_memory_insert_payload(
         finalize_revision: bool,
     ) -> dict[str, Any]:
         """Map a frozen source chunk to the strict V4MemoryInsertRequest shape."""
+        if len(chunk.content) > MESA_V4_EVIDENCE_SPAN_MAX_CHARS:
+            raise ValueError(
+                "Source chunk exceeds the MESA V4 evidence span contract; "
+                "rebuild the deterministic chunk plan before publishing"
+            )
         metadata = {
             **chunk.metadata,
             "mesa_data_chunk_type": chunk.chunk_type,
@@ -410,7 +416,7 @@ def build_memory_insert_payload(
             "title": chunk.title or f"Document {chunk.document_id} chunk {chunk.ordinal}",
             "source_ref": source_ref,
             "content": chunk.content,
-            "evidence_span": "",
+            "evidence_span": chunk.content,
             "revision_number": int(metadata.get("revision_number", 1)),
             "chunk_ordinal": chunk.ordinal,
             "finalize_revision": finalize_revision,

src/mesa_legal_data/publisher/models.py

@@ -3,6 +3,8 @@
 
 from pydantic import BaseModel, ConfigDict, Field
 
+MESA_V4_EVIDENCE_SPAN_MAX_CHARS = 4096
+
 
 class DeliveryStatus(str, Enum):
     PLANNED = "PLANNED"

src/mesa_legal_data/release/builder.py

@@ -564,7 +564,9 @@ def build_qualification_release(
         validate_release_id(release_id)
 
     if release_id == "release-20260831T145932Z":
-        raise ReleaseBuildError("Cannot overwrite immutable historical qualification release 'release-20260831T145932Z'")
+        raise ReleaseBuildError(
+            "Cannot overwrite immutable historical qualification release 'release-20260831T145932Z'"
+        )
 
     final_dir = data_root / "releases" / release_id
     if final_dir.exists():
@@ -597,7 +599,12 @@ def build_qualification_release(
         # Reconstruct base index deterministically from base payloads
         with open(building_data_dir / "release-index.jsonl", "w", encoding="utf-8") as idx_out:
             for r_type in ["legislation", "article", "decision", "citation"]:
-                plural_map = {"legislation": "legislation.jsonl", "article": "articles.jsonl", "decision": "decisions.jsonl", "citation": "citations.jsonl"}
+                plural_map = {
+                    "legislation": "legislation.jsonl",
+                    "article": "articles.jsonl",
+                    "decision": "decisions.jsonl",
+                    "citation": "citations.jsonl",
+                }
                 target_f = building_data_dir / plural_map[r_type]
                 if not target_f.exists():
                     continue
@@ -608,7 +615,9 @@ def build_qualification_release(
                             continue
                         rec = json.loads(line_str)
                         p_sha = hashlib.sha256((line_str + "\n").encode("utf-8")).hexdigest()
-                        v_id = rec.get("version", {}).get("version_id") or rec.get("legislation_version_id") or rec["id"]
+                        v_id = (
+                            rec.get("version", {}).get("version_id") or rec.get("legislation_version_id") or rec["id"]
+                        )
                         d_id = rec.get("legislation_id") or rec["id"]
                         idx_out.write(
                             json_str_deterministic(
@@ -660,7 +669,7 @@ def build_qualification_release(
     # 6. Append fixture canonical records to release files
     leg_out = building_data_dir / "legislation.jsonl"
     art_out = building_data_dir / "articles.jsonl"
-    idx_out = building_data_dir / "release-index.jsonl"
+    release_index_path = building_data_dir / "release-index.jsonl"
 
     with open(leg_out, "a", encoding="utf-8") as f_leg:
         for rec in c_records:
@@ -672,7 +681,7 @@ def build_qualification_release(
             if rec["record_type"] == "article":
                 f_art.write(json_str_deterministic(rec) + "\n")
 
-    with open(idx_out, "a", encoding="utf-8") as f_idx:
+    with open(release_index_path, "a", encoding="utf-8") as f_idx:
         for entry in r_entries:
             f_idx.write(json_str_deterministic(entry) + "\n")
 

src/mesa_legal_data/release/qualification_fixtures.py

@@ -570,7 +570,9 @@ def build_qualification_fixtures(
             }
             canonical_records.append(leg_record)
             leg_payload_sha = hashlib.sha256(
-                (json.dumps(leg_record, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
+                (json.dumps(leg_record, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode(
+                    "utf-8"
+                )
             ).hexdigest()
             release_index_entries.append(
                 {
@@ -681,9 +683,7 @@ def validate_qualification_fixtures(
     }
     missing_scope = sorted(required_scope_fields - qualification_scope.keys())
     if missing_scope:
-        raise QualificationFixtureError(
-            f"qualification_scope is missing required fields: {missing_scope}"
-        )
+        raise QualificationFixtureError(f"qualification_scope is missing required fields: {missing_scope}")
     tenant_id = qualification_scope["tenant_id"]
     workspace_id = qualification_scope["workspace_id"]
     dataset_ids = qualification_scope["dataset_ids"]
@@ -718,9 +718,7 @@ def validate_qualification_fixtures(
     }
     missing_auth = sorted(required_authority_fields - scope_test_authority.keys())
     if missing_auth:
-        raise QualificationFixtureError(
-            f"scope_test_authority is missing required fields: {missing_auth}"
-        )
+        raise QualificationFixtureError(f"scope_test_authority is missing required fields: {missing_auth}")
 
     forbidden_tenant = scope_test_authority["forbidden_tenant"]
     forbidden_dataset = scope_test_authority["forbidden_dataset"]
@@ -759,15 +757,11 @@ def validate_qualification_fixtures(
                 else QualificationIdentityRow.model_validate(raw_row)
             )
         except ValidationError as exc:
-            raise QualificationFixtureError(
-                f"invalid identity map row at line {idx}: {exc}"
-            ) from exc
+            raise QualificationFixtureError(f"invalid identity map row at line {idx}: {exc}") from exc
 
         pair = (row.mesa_chunk_id, row.source_chunk_id)
         if pair in seen_chunk_pairs:
-            raise QualificationFixtureError(
-                f"duplicate identity mapping at line {idx}: {pair!r}"
-            )
+            raise QualificationFixtureError(f"duplicate identity mapping at line {idx}: {pair!r}")
         seen_chunk_pairs.add(pair)
 
         rows_by_source_chunk.setdefault(row.source_chunk_id, []).append(row)
@@ -776,9 +770,7 @@ def validate_qualification_fixtures(
 
     # Authorized document presence in identity map
     if authorized_document not in rows_by_document_id:
-        raise QualificationFixtureError(
-            f"authorized_document {authorized_document!r} is absent from identity map"
-        )
+        raise QualificationFixtureError(f"authorized_document {authorized_document!r} is absent from identity map")
 
     # 4. Check case coverage
     if not isinstance(case_evidence_fixtures, dict):
@@ -801,29 +793,21 @@ def validate_qualification_fixtures(
             or not fixtures
             or any(not isinstance(fid, str) or not fid.strip() for fid in fixtures)
         ):
-            raise QualificationFixtureError(
-                f"case {case_id} requires at least one non-empty fixture ID"
-            )
+            raise QualificationFixtureError(f"case {case_id} requires at least one non-empty fixture ID")
         if len(fixtures) != len(set(fixtures)):
-            raise QualificationFixtureError(
-                f"case {case_id} contains duplicate fixture IDs"
-            )
+            raise QualificationFixtureError(f"case {case_id} contains duplicate fixture IDs")
         all_fixture_ids.extend(fixtures)
 
     if len(all_fixture_ids) != len(set(all_fixture_ids)):
-        raise QualificationFixtureError(
-            "fixture IDs must be unique across all Phase 7 cases"
-        )
+        raise QualificationFixtureError("fixture IDs must be unique across all Phase 7 cases")
 
     if not isinstance(corpus_fixtures, dict):
         raise QualificationFixtureError("corpus_fixtures must be an object")
 
     if set(corpus_fixtures.keys()) != set(all_fixture_ids):
         missing = sorted(set(all_fixture_ids) - set(corpus_fixtures.keys()))
         unreferenced = sorted(set(corpus_fixtures.keys()) - set(all_fixture_ids))
-        raise QualificationFixtureError(
-            f"corpus_fixtures mismatch; missing={missing}, unreferenced={unreferenced}"
-        )
+        raise QualificationFixtureError(f"corpus_fixtures mismatch; missing={missing}, unreferenced={unreferenced}")
 
     # 5. Semantic validation of each fixture
     for case_id in REQUIRED_SCOPE_CASE_IDS:
@@ -851,9 +835,7 @@ def validate_qualification_fixtures(
                     f"fixture {fixture_id!r} has identity_type {identity_type!r}; case {case_id} requires {expected_type!r}"
                 )
             if not isinstance(source_chunk_id, str) or not source_chunk_id:
-                raise QualificationFixtureError(
-                    f"fixture {fixture_id!r} lacks source_chunk_id binding"
-                )
+                raise QualificationFixtureError(f"fixture {fixture_id!r} lacks source_chunk_id binding")
 
             # Resolve bound rows
             if source_chunk_id not in rows_by_source_chunk:
@@ -863,9 +845,7 @@ def validate_qualification_fixtures(
 
             source_rows = rows_by_source_chunk[source_chunk_id]
             if identity_type == "chunk":
-                bound_rows = [
-                    r for r in source_rows if fixture_id in {r.mesa_chunk_id, r.source_chunk_id}
-                ]
+                bound_rows = [r for r in source_rows if fixture_id in {r.mesa_chunk_id, r.source_chunk_id}]
             elif identity_type == "evidence":
                 bound_rows = [r for r in source_rows if r.evidence_id == fixture_id]
             elif identity_type == "catalog":
@@ -923,14 +903,10 @@ def validate_qualification_fixtures(
                     )
             elif case_id == "wrong_jurisdiction_search":
                 if record.get("jurisdiction") in {None, "TR"}:
-                    raise QualificationFixtureError(
-                        f"wrong-jurisdiction fixture {fixture_id!r} is not outside TR"
-                    )
+                    raise QualificationFixtureError(f"wrong-jurisdiction fixture {fixture_id!r} is not outside TR")
             elif case_id == "stale_version_search":
                 if record.get("is_current") is not False:
-                    raise QualificationFixtureError(
-                        f"stale fixture {fixture_id!r} is not marked non-current"
-                    )
+                    raise QualificationFixtureError(f"stale fixture {fixture_id!r} is not marked non-current")
             elif case_id == "effective_date_boundary_search":
                 try:
                     vf = datetime.fromisoformat(str(record["valid_from"]).replace("Z", "+00:00"))
@@ -963,16 +939,10 @@ def load_baseline_identity_map_rows(
 ) -> list[dict[str, Any]]:
     """Loads and validates the 5,721 baseline identity map rows."""
     if baseline_path is None:
-        baseline_path = (
-            Path(__file__).parent.parent
-            / "resources"
-            / "frozen_qualification_baseline_identity_map.jsonl"
-        )
+        baseline_path = Path(__file__).parent.parent / "resources" / "frozen_qualification_baseline_identity_map.jsonl"
 
     if not baseline_path.exists():
-        raise QualificationFixtureError(
-            f"baseline identity map file not found: {baseline_path}"
-        )
+        raise QualificationFixtureError(f"baseline identity map file not found: {baseline_path}")
 
     raw_bytes = baseline_path.read_bytes()
     expected_sha256 = "960c4085e40082e88a5e1baf680c58fd2c5d8e00cefd2aaa180cb79e8338336d"
@@ -992,8 +962,6 @@ def load_baseline_identity_map_rows(
         rows.append(row_dict)
 
     if len(rows) != 5721:
-        raise QualificationFixtureError(
-            f"expected exactly 5721 baseline identity map rows, got {len(rows)}"
-        )
+        raise QualificationFixtureError(f"expected exactly 5721 baseline identity map rows, got {len(rows)}")
 
     return rows

src/mesa_legal_data/release/verifier.py

@@ -173,9 +173,13 @@ def verify_release_directory(release_dir: Path, expected_release_id: str | None
 
     if fixtures_file.exists() or identity_map_file.exists():
         if not fixtures_file.exists():
-            raise ReleaseVerificationError("identity_map.jsonl is present but data/qualification_fixtures.json is missing")
+            raise ReleaseVerificationError(
+                "identity_map.jsonl is present but data/qualification_fixtures.json is missing"
+            )
         if not identity_map_file.exists():
-            raise ReleaseVerificationError("qualification_fixtures.json is present but data/identity_map.jsonl is missing")
+            raise ReleaseVerificationError(
+                "qualification_fixtures.json is present but data/identity_map.jsonl is missing"
+            )
 
         try:
             with open(fixtures_file, "r", encoding="utf-8") as f:
@@ -219,9 +223,7 @@ def verify_release_directory(release_dir: Path, expected_release_id: str | None
                 identity_map_rows=identity_rows,
             )
         except QualificationFixtureError as exc:
-            raise ReleaseVerificationError(
-                f"Phase 7 qualification fixture semantic validation failed: {exc}"
-            ) from exc
+            raise ReleaseVerificationError(f"Phase 7 qualification fixture semantic validation failed: {exc}") from exc
 
         expected_map_sha = release_meta.get("identity_map_sha256")
         if expected_map_sha: