diff --git a/pyproject.toml b/pyproject.toml index bf3bfde..28e5e7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,4 +59,5 @@ ignore = ["BLE001", "B008", "DTZ007", "E501", "W293", "W291"] [tool.pytest.ini_options] markers = [ "scale: Large-scale or heavy benchmark tests processing thousands of records", + "cross_repo: Optional compatibility test against a checked-out MESA source tree", ] diff --git a/src/mesa_legal_data/publisher/chunker.py b/src/mesa_legal_data/publisher/chunker.py index 0814d2e..53c9bc6 100644 --- a/src/mesa_legal_data/publisher/chunker.py +++ b/src/mesa_legal_data/publisher/chunker.py @@ -1,7 +1,7 @@ from typing import Any from mesa_legal_data.publisher.hashing import calculate_content_hash -from mesa_legal_data.publisher.models import SourceChunk +from mesa_legal_data.publisher.models import MESA_V4_EVIDENCE_SPAN_MAX_CHARS, SourceChunk def _split_oversized_text( @@ -82,6 +82,12 @@ def plan_source_chunks( if not canonical_text or not canonical_text.strip(): return [] + # MESA's current V4 insert contract accepts at most 4,096 characters in + # ``evidence_span``. Every source chunk becomes the complete evidence + # payload for its deterministic fallback assertion, so the semantic limit + # is stricter than the 32,768-character transport-content limit. + chunk_limit_chars = min(content_limit_chars, MESA_V4_EVIDENCE_SPAN_MAX_CHARS) + # Auto-resolve char_start and char_end if content is provided search_cursor = 0 for r in records: @@ -105,7 +111,7 @@ def plan_source_chunks( if not article_records: # Generic text without article records (e.g. unsegmented or single decision) - splits = _split_oversized_text(canonical_text, 0, content_limit_chars) + splits = _split_oversized_text(canonical_text, 0, chunk_limit_chars) for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): chunk_id = ( f"{version_id}:chunk:{ordinal_counter:04d}" @@ -136,7 +142,7 @@ def plan_source_chunks( if first_art_start > 0: preamble_text = canonical_text[:first_art_start] if preamble_text.strip(): - splits = _split_oversized_text(preamble_text, 0, content_limit_chars) + splits = _split_oversized_text(preamble_text, 0, chunk_limit_chars) for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): chunk_id = ( f"{version_id}:chunk:{ordinal_counter:04d}" @@ -170,7 +176,7 @@ def plan_source_chunks( art_num = art.get("article_number", "") art_title = art.get("title") or (f"Madde {art_num}" if art_num else "Madde") - splits = _split_oversized_text(art_text, a_start, content_limit_chars) + splits = _split_oversized_text(art_text, a_start, chunk_limit_chars) for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): chunk_id = ( f"{version_id}:chunk:{ordinal_counter:04d}" @@ -205,7 +211,7 @@ def plan_source_chunks( if last_art_end < len(canonical_text): annex_text = canonical_text[last_art_end:] if annex_text.strip(): - splits = _split_oversized_text(annex_text, last_art_end, content_limit_chars) + splits = _split_oversized_text(annex_text, last_art_end, chunk_limit_chars) for piece_idx, (c_start, c_end, piece_text) in enumerate(splits, start=1): chunk_id = ( f"{version_id}:chunk:{ordinal_counter:04d}" diff --git a/src/mesa_legal_data/publisher/client.py b/src/mesa_legal_data/publisher/client.py index 37c9daa..5d5dc77 100644 --- a/src/mesa_legal_data/publisher/client.py +++ b/src/mesa_legal_data/publisher/client.py @@ -6,6 +6,7 @@ import httpx from mesa_legal_data.publisher.models import ( + MESA_V4_EVIDENCE_SPAN_MAX_CHARS, MesaTargetSettings, MutationState, PreflightCheckItem, @@ -391,6 +392,11 @@ def build_memory_insert_payload( finalize_revision: bool, ) -> dict[str, Any]: """Map a frozen source chunk to the strict V4MemoryInsertRequest shape.""" + if len(chunk.content) > MESA_V4_EVIDENCE_SPAN_MAX_CHARS: + raise ValueError( + "Source chunk exceeds the MESA V4 evidence span contract; " + "rebuild the deterministic chunk plan before publishing" + ) metadata = { **chunk.metadata, "mesa_data_chunk_type": chunk.chunk_type, @@ -410,7 +416,7 @@ def build_memory_insert_payload( "title": chunk.title or f"Document {chunk.document_id} chunk {chunk.ordinal}", "source_ref": source_ref, "content": chunk.content, - "evidence_span": "", + "evidence_span": chunk.content, "revision_number": int(metadata.get("revision_number", 1)), "chunk_ordinal": chunk.ordinal, "finalize_revision": finalize_revision, diff --git a/src/mesa_legal_data/publisher/models.py b/src/mesa_legal_data/publisher/models.py index f8118c8..71d0c4a 100644 --- a/src/mesa_legal_data/publisher/models.py +++ b/src/mesa_legal_data/publisher/models.py @@ -3,6 +3,8 @@ from pydantic import BaseModel, ConfigDict, Field +MESA_V4_EVIDENCE_SPAN_MAX_CHARS = 4096 + class DeliveryStatus(str, Enum): PLANNED = "PLANNED" diff --git a/src/mesa_legal_data/release/builder.py b/src/mesa_legal_data/release/builder.py index 7afb725..7572d10 100644 --- a/src/mesa_legal_data/release/builder.py +++ b/src/mesa_legal_data/release/builder.py @@ -564,7 +564,9 @@ def build_qualification_release( validate_release_id(release_id) if release_id == "release-20260831T145932Z": - raise ReleaseBuildError("Cannot overwrite immutable historical qualification release 'release-20260831T145932Z'") + raise ReleaseBuildError( + "Cannot overwrite immutable historical qualification release 'release-20260831T145932Z'" + ) final_dir = data_root / "releases" / release_id if final_dir.exists(): @@ -597,7 +599,12 @@ def build_qualification_release( # Reconstruct base index deterministically from base payloads with open(building_data_dir / "release-index.jsonl", "w", encoding="utf-8") as idx_out: for r_type in ["legislation", "article", "decision", "citation"]: - plural_map = {"legislation": "legislation.jsonl", "article": "articles.jsonl", "decision": "decisions.jsonl", "citation": "citations.jsonl"} + plural_map = { + "legislation": "legislation.jsonl", + "article": "articles.jsonl", + "decision": "decisions.jsonl", + "citation": "citations.jsonl", + } target_f = building_data_dir / plural_map[r_type] if not target_f.exists(): continue @@ -608,7 +615,9 @@ def build_qualification_release( continue rec = json.loads(line_str) p_sha = hashlib.sha256((line_str + "\n").encode("utf-8")).hexdigest() - v_id = rec.get("version", {}).get("version_id") or rec.get("legislation_version_id") or rec["id"] + v_id = ( + rec.get("version", {}).get("version_id") or rec.get("legislation_version_id") or rec["id"] + ) d_id = rec.get("legislation_id") or rec["id"] idx_out.write( json_str_deterministic( @@ -660,7 +669,7 @@ def build_qualification_release( # 6. Append fixture canonical records to release files leg_out = building_data_dir / "legislation.jsonl" art_out = building_data_dir / "articles.jsonl" - idx_out = building_data_dir / "release-index.jsonl" + release_index_path = building_data_dir / "release-index.jsonl" with open(leg_out, "a", encoding="utf-8") as f_leg: for rec in c_records: @@ -672,7 +681,7 @@ def build_qualification_release( if rec["record_type"] == "article": f_art.write(json_str_deterministic(rec) + "\n") - with open(idx_out, "a", encoding="utf-8") as f_idx: + with open(release_index_path, "a", encoding="utf-8") as f_idx: for entry in r_entries: f_idx.write(json_str_deterministic(entry) + "\n") diff --git a/src/mesa_legal_data/release/qualification_fixtures.py b/src/mesa_legal_data/release/qualification_fixtures.py index aa72cfc..1cf7fd6 100644 --- a/src/mesa_legal_data/release/qualification_fixtures.py +++ b/src/mesa_legal_data/release/qualification_fixtures.py @@ -570,7 +570,9 @@ def build_qualification_fixtures( } canonical_records.append(leg_record) leg_payload_sha = hashlib.sha256( - (json.dumps(leg_record, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + (json.dumps(leg_record, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) ).hexdigest() release_index_entries.append( { @@ -681,9 +683,7 @@ def validate_qualification_fixtures( } missing_scope = sorted(required_scope_fields - qualification_scope.keys()) if missing_scope: - raise QualificationFixtureError( - f"qualification_scope is missing required fields: {missing_scope}" - ) + raise QualificationFixtureError(f"qualification_scope is missing required fields: {missing_scope}") tenant_id = qualification_scope["tenant_id"] workspace_id = qualification_scope["workspace_id"] dataset_ids = qualification_scope["dataset_ids"] @@ -718,9 +718,7 @@ def validate_qualification_fixtures( } missing_auth = sorted(required_authority_fields - scope_test_authority.keys()) if missing_auth: - raise QualificationFixtureError( - f"scope_test_authority is missing required fields: {missing_auth}" - ) + raise QualificationFixtureError(f"scope_test_authority is missing required fields: {missing_auth}") forbidden_tenant = scope_test_authority["forbidden_tenant"] forbidden_dataset = scope_test_authority["forbidden_dataset"] @@ -759,15 +757,11 @@ def validate_qualification_fixtures( else QualificationIdentityRow.model_validate(raw_row) ) except ValidationError as exc: - raise QualificationFixtureError( - f"invalid identity map row at line {idx}: {exc}" - ) from exc + raise QualificationFixtureError(f"invalid identity map row at line {idx}: {exc}") from exc pair = (row.mesa_chunk_id, row.source_chunk_id) if pair in seen_chunk_pairs: - raise QualificationFixtureError( - f"duplicate identity mapping at line {idx}: {pair!r}" - ) + raise QualificationFixtureError(f"duplicate identity mapping at line {idx}: {pair!r}") seen_chunk_pairs.add(pair) rows_by_source_chunk.setdefault(row.source_chunk_id, []).append(row) @@ -776,9 +770,7 @@ def validate_qualification_fixtures( # Authorized document presence in identity map if authorized_document not in rows_by_document_id: - raise QualificationFixtureError( - f"authorized_document {authorized_document!r} is absent from identity map" - ) + raise QualificationFixtureError(f"authorized_document {authorized_document!r} is absent from identity map") # 4. Check case coverage if not isinstance(case_evidence_fixtures, dict): @@ -801,19 +793,13 @@ def validate_qualification_fixtures( or not fixtures or any(not isinstance(fid, str) or not fid.strip() for fid in fixtures) ): - raise QualificationFixtureError( - f"case {case_id} requires at least one non-empty fixture ID" - ) + raise QualificationFixtureError(f"case {case_id} requires at least one non-empty fixture ID") if len(fixtures) != len(set(fixtures)): - raise QualificationFixtureError( - f"case {case_id} contains duplicate fixture IDs" - ) + raise QualificationFixtureError(f"case {case_id} contains duplicate fixture IDs") all_fixture_ids.extend(fixtures) if len(all_fixture_ids) != len(set(all_fixture_ids)): - raise QualificationFixtureError( - "fixture IDs must be unique across all Phase 7 cases" - ) + raise QualificationFixtureError("fixture IDs must be unique across all Phase 7 cases") if not isinstance(corpus_fixtures, dict): raise QualificationFixtureError("corpus_fixtures must be an object") @@ -821,9 +807,7 @@ def validate_qualification_fixtures( if set(corpus_fixtures.keys()) != set(all_fixture_ids): missing = sorted(set(all_fixture_ids) - set(corpus_fixtures.keys())) unreferenced = sorted(set(corpus_fixtures.keys()) - set(all_fixture_ids)) - raise QualificationFixtureError( - f"corpus_fixtures mismatch; missing={missing}, unreferenced={unreferenced}" - ) + raise QualificationFixtureError(f"corpus_fixtures mismatch; missing={missing}, unreferenced={unreferenced}") # 5. Semantic validation of each fixture for case_id in REQUIRED_SCOPE_CASE_IDS: @@ -851,9 +835,7 @@ def validate_qualification_fixtures( f"fixture {fixture_id!r} has identity_type {identity_type!r}; case {case_id} requires {expected_type!r}" ) if not isinstance(source_chunk_id, str) or not source_chunk_id: - raise QualificationFixtureError( - f"fixture {fixture_id!r} lacks source_chunk_id binding" - ) + raise QualificationFixtureError(f"fixture {fixture_id!r} lacks source_chunk_id binding") # Resolve bound rows if source_chunk_id not in rows_by_source_chunk: @@ -863,9 +845,7 @@ def validate_qualification_fixtures( source_rows = rows_by_source_chunk[source_chunk_id] if identity_type == "chunk": - bound_rows = [ - r for r in source_rows if fixture_id in {r.mesa_chunk_id, r.source_chunk_id} - ] + bound_rows = [r for r in source_rows if fixture_id in {r.mesa_chunk_id, r.source_chunk_id}] elif identity_type == "evidence": bound_rows = [r for r in source_rows if r.evidence_id == fixture_id] elif identity_type == "catalog": @@ -923,14 +903,10 @@ def validate_qualification_fixtures( ) elif case_id == "wrong_jurisdiction_search": if record.get("jurisdiction") in {None, "TR"}: - raise QualificationFixtureError( - f"wrong-jurisdiction fixture {fixture_id!r} is not outside TR" - ) + raise QualificationFixtureError(f"wrong-jurisdiction fixture {fixture_id!r} is not outside TR") elif case_id == "stale_version_search": if record.get("is_current") is not False: - raise QualificationFixtureError( - f"stale fixture {fixture_id!r} is not marked non-current" - ) + raise QualificationFixtureError(f"stale fixture {fixture_id!r} is not marked non-current") elif case_id == "effective_date_boundary_search": try: vf = datetime.fromisoformat(str(record["valid_from"]).replace("Z", "+00:00")) @@ -963,16 +939,10 @@ def load_baseline_identity_map_rows( ) -> list[dict[str, Any]]: """Loads and validates the 5,721 baseline identity map rows.""" if baseline_path is None: - baseline_path = ( - Path(__file__).parent.parent - / "resources" - / "frozen_qualification_baseline_identity_map.jsonl" - ) + baseline_path = Path(__file__).parent.parent / "resources" / "frozen_qualification_baseline_identity_map.jsonl" if not baseline_path.exists(): - raise QualificationFixtureError( - f"baseline identity map file not found: {baseline_path}" - ) + raise QualificationFixtureError(f"baseline identity map file not found: {baseline_path}") raw_bytes = baseline_path.read_bytes() expected_sha256 = "960c4085e40082e88a5e1baf680c58fd2c5d8e00cefd2aaa180cb79e8338336d" @@ -992,8 +962,6 @@ def load_baseline_identity_map_rows( rows.append(row_dict) if len(rows) != 5721: - raise QualificationFixtureError( - f"expected exactly 5721 baseline identity map rows, got {len(rows)}" - ) + raise QualificationFixtureError(f"expected exactly 5721 baseline identity map rows, got {len(rows)}") return rows diff --git a/src/mesa_legal_data/release/verifier.py b/src/mesa_legal_data/release/verifier.py index 1e8fb6b..89b8646 100644 --- a/src/mesa_legal_data/release/verifier.py +++ b/src/mesa_legal_data/release/verifier.py @@ -173,9 +173,13 @@ def verify_release_directory(release_dir: Path, expected_release_id: str | None if fixtures_file.exists() or identity_map_file.exists(): if not fixtures_file.exists(): - raise ReleaseVerificationError("identity_map.jsonl is present but data/qualification_fixtures.json is missing") + raise ReleaseVerificationError( + "identity_map.jsonl is present but data/qualification_fixtures.json is missing" + ) if not identity_map_file.exists(): - raise ReleaseVerificationError("qualification_fixtures.json is present but data/identity_map.jsonl is missing") + raise ReleaseVerificationError( + "qualification_fixtures.json is present but data/identity_map.jsonl is missing" + ) try: with open(fixtures_file, "r", encoding="utf-8") as f: @@ -219,9 +223,7 @@ def verify_release_directory(release_dir: Path, expected_release_id: str | None identity_map_rows=identity_rows, ) except QualificationFixtureError as exc: - raise ReleaseVerificationError( - f"Phase 7 qualification fixture semantic validation failed: {exc}" - ) from exc + raise ReleaseVerificationError(f"Phase 7 qualification fixture semantic validation failed: {exc}") from exc expected_map_sha = release_meta.get("identity_map_sha256") if expected_map_sha: diff --git a/tests/integration/test_mesa_v4_vector_cross_repo.py b/tests/integration/test_mesa_v4_vector_cross_repo.py new file mode 100644 index 0000000..61d5b06 --- /dev/null +++ b/tests/integration/test_mesa_v4_vector_cross_repo.py @@ -0,0 +1,276 @@ +"""Optional real-storage compatibility proof against the current MESA checkout.""" + +from __future__ import annotations + +import datetime +import hashlib +import json +import os +import sys +from pathlib import Path + +import pytest + +# The authoritative MESA checkout currently uses Python 3.10 while MESA_Data +# itself requires 3.11+. Supply the standard 3.11 UTC alias so the newer +# producer package can be imported by MESA's dependency environment. +if not hasattr(datetime, "UTC"): + datetime.UTC = datetime.timezone.utc # type: ignore[attr-defined] + +from mesa_legal_data.publisher.chunker import plan_source_chunks +from mesa_legal_data.publisher.client import MesaClient +from mesa_legal_data.publisher.models import MesaTargetSettings + +MESA_SOURCE_ROOT = os.environ.get("MESA_SOURCE_ROOT") +if not MESA_SOURCE_ROOT: + pytest.skip("set MESA_SOURCE_ROOT to run cross-repository compatibility", allow_module_level=True) + +mesa_root = Path(MESA_SOURCE_ROOT).resolve() +if not (mesa_root / "mesa_storage" / "representation.py").is_file(): + pytest.skip("MESA_SOURCE_ROOT is not a MESA source checkout", allow_module_level=True) +sys.path.insert(0, str(mesa_root)) + +from mesa_memory.config import config, configured_embedding_identity # noqa: E402 +from mesa_memory.embedding.service import ( # noqa: E402 + EmbeddingIdentity, + EmbeddingService, +) +from mesa_storage.dao import MemoryDAO # noqa: E402 +from mesa_storage.kuzu_provider import KuzuGraphProvider # noqa: E402 +from mesa_storage.kuzu_setup import initialize_schema_artifact # noqa: E402 +from mesa_storage.representation import ( # noqa: E402 + V4_VECTOR_REPRESENTATION_VERSION, +) +from mesa_storage.schemas import initialize_schema # noqa: E402 +from mesa_storage.sqlite_engine import AsyncEngine # noqa: E402 +from mesa_storage.vector_engine import VectorEngine # noqa: E402 +from mesa_workers.ingestion_worker import process_cold_path # noqa: E402 +from mesa_workers.projection_worker import process_projection_outbox_once # noqa: E402 + + +def _semantic_vector(text: str) -> list[float]: + """Small deterministic semantic fixture in the configured vector dimension.""" + folded = text.casefold() + vector = [0.0] * 768 + if "gelmeden görev" in folded or "uzaktan mesai" in folded: + vector[0] = 1.0 + elif "arşiv kayıtları" in folded or "saklama süresi" in folded: + vector[1] = 1.0 + else: + vector[2] = 1.0 + return vector + + +@pytest.mark.cross_repo +@pytest.mark.asyncio +async def test_mesa_data_chunks_produce_current_mesa_vector_artifacts(tmp_path: Path) -> None: + runtime_identity = configured_embedding_identity() + assert runtime_identity.provider == "local" + assert runtime_identity.model == "magibu/embeddingmagibu-200m" + assert runtime_identity.dimension == 768 + + identity = EmbeddingIdentity( + provider=runtime_identity.provider, + model=runtime_identity.model, + version=runtime_identity.version, + dimension=runtime_identity.dimension, + normalized=runtime_identity.normalized, + model_revision=runtime_identity.model_revision, + ) + service = EmbeddingService( + identity=identity, + provider_fn=_semantic_vector, + query_provider_fn=_semantic_vector, + ) + sql = AsyncEngine(str(tmp_path / "mesa.sqlite")) + vector = VectorEngine( + str(tmp_path / "vectors.lance"), + max_workers=1, + embedding_service=service, + ) + graph_path = tmp_path / "graph" + await sql.initialize() + await initialize_schema(sql) + await vector.initialize() + initialize_schema_artifact(str(graph_path)) + graph = KuzuGraphProvider(str(graph_path), max_workers=1) + await graph.initialize() + dao = MemoryDAO(sqlite_engine=sql, vector_engine=vector, graph_provider=graph) + + settings = MesaTargetSettings( + tenant_id="tenant-vector", + workspace_id="workspace-vector", + dataset_id="dataset-vector", + agent_id="agent-vector", + ) + client = MesaClient(settings, api_key="test-only") + target_article = "MADDE 1- " + ("Genel açıklama. " * 18) + "Çalışanlar kurum binasına gelmeden görev yapabilir." + distractor_article = "MADDE 2- Arşiv kayıtları yasal saklama süresi boyunca korunur." + canonical_text = f"{target_article}\n\n{distractor_article}" + second_start = len(target_article) + 2 + chunks = plan_source_chunks( + document_id="law-remote-work", + version_id="law-remote-work:v1", + canonical_text=canonical_text, + records=[ + { + "record_id": "article-1", + "record_type": "article", + "article_number": "1", + "title": "Madde 1", + "char_start": 0, + "char_end": len(target_article), + "ordinal": 1, + }, + { + "record_id": "article-2", + "record_type": "article", + "article_number": "2", + "title": "Madde 2", + "char_start": second_start, + "char_end": len(canonical_text), + "ordinal": 2, + }, + ], + ) + assert len(chunks) == 2 + assert all(len(chunk.content) <= 4096 for chunk in chunks) + + try: + await dao.ensure_v4_catalog_scope( + tenant_id=settings.tenant_id, + workspace_id=settings.workspace_id, + dataset_id=settings.dataset_id, + ) + admitted: list[tuple[int, str]] = [] + for index, chunk in enumerate(chunks): + chunk.metadata.update( + { + "release_id": "release-vector-v1", + "revision_number": 1, + "source_url": "https://authority.example/law/remote-work", + "source_id": "authority-law-remote-work", + "artifact_sha256": "a" * 64, + "is_final_chunk": index == len(chunks) - 1, + } + ) + payload = client.build_memory_insert_payload( + chunk, + session_id="session-vector", + idempotency_key=f"vector-{index}", + finalize_revision=index == len(chunks) - 1, + ) + assert payload["evidence_span"] == payload["content"] + admission_args = dict( + tenant_id=settings.tenant_id, + workspace_id=settings.workspace_id, + dataset_id=settings.dataset_id, + agent_id=settings.agent_id, + session_id=payload["session_id"], + document_id=payload["document_id"], + revision_id=payload["revision_id"], + chunk_id=payload["chunk_id"], + title=payload["title"], + content_payload=payload["content"], + source_ref=payload["source_ref"], + evidence_span=payload["evidence_span"], + revision_number=payload["revision_number"], + chunk_ordinal=payload["chunk_ordinal"], + supersedes_revision_id=payload["supersedes_revision_id"], + metadata=payload["metadata"], + embedding_provider=identity.provider, + embedding_model=identity.model, + embedding_version=identity.version, + embedding_dimension=identity.dimension, + embedding_space_id=identity.embedding_space_id, + embedding_model_revision=identity.model_revision, + embedding_normalized=identity.normalized, + policy=config.queue_admission_policy, + validation_mode=0, + idempotency_key=payload["idempotency_key"], + payload_hash=hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest(), + finalize_revision=payload["finalize_revision"], + ) + result = await dao.admit_v4_memory(**admission_args) + duplicate = await dao.admit_v4_memory(**admission_args) + assert duplicate == { + "outcome": "DUPLICATE", + "response": result["response"], + } + mutation_id = result["response"]["mutation_id"] + async with sql.connection() as db: + async with db.execute( + "SELECT raw_log_id FROM memory_mutations WHERE mutation_id = ?", + (mutation_id,), + ) as cursor: + raw_log_id = int((await cursor.fetchone())[0]) + admitted.append((raw_log_id, mutation_id)) + + for raw_log_id, _mutation_id in admitted: + await process_cold_path( + raw_log_id, + settings.agent_id, + dao, + model_processing_enabled=False, + ) + + for _ in range(12): + outcome = await process_projection_outbox_once(dao, worker_id="cross-repo-vector", limit=1) + if outcome["claimed"] == 0: + break + + async with sql.connection() as db: + async with db.execute( + "SELECT r.registry_id, r.physical_artifact_id, r.metadata_json, " + "s.dataset_id, s.document_id, s.revision_id, s.chunk_id, " + "s.source_ref, s.evidence_span " + "FROM artifact_registry r JOIN artifact_sources s " + "ON s.registry_id = r.registry_id " + "WHERE r.artifact_kind = 'ASSERTION_VECTOR' AND r.state = 'ACTIVE' " + "ORDER BY s.chunk_id" + ) as cursor: + registry_rows = await cursor.fetchall() + async with db.execute("SELECT assertion_id FROM v4_assertions WHERE status = 'ACTIVE'") as cursor: + assertion_ids = {str(row[0]) for row in await cursor.fetchall()} + + assert len(registry_rows) == len(chunks) == 2 + registry_vector_ids = {str(row[1]) for row in registry_rows} + assert registry_vector_ids == assertion_ids + assert await vector.get_active_node_ids(settings.agent_id) == assertion_ids + embeddings = await vector.get_all_embeddings(agent_id=settings.agent_id) + assert len(embeddings) == 2 + assert {len(embedding) for embedding in embeddings} == {identity.dimension} + for row in registry_rows: + metadata = json.loads(row[2]) + assert metadata == { + **identity.as_dict(), + "embedding_dimension": identity.dimension, + "representation_version": V4_VECTOR_REPRESENTATION_VERSION, + } + assert row[3] + assert row[4] + assert row[5] + assert row[6] + assert row[7] == "https://authority.example/law/remote-work" + assert row[8] + + diagnostics: dict = {} + results = await dao.search_v4_memory( + tenant_id=settings.tenant_id, + agent_id=settings.agent_id, + dataset_ids=[settings.dataset_id], + query="Personel için uzaktan mesai yetkisi", + limit=10, + certification_metadata=diagnostics, + ) + assert results + semantic_hit = next(result for result in results if "gelmeden görev" in result["evidence_span"].casefold()) + assert "vector" in semantic_hit["retrieval_provenance"]["origins"] + assert semantic_hit["retrieval_provenance"]["vector"]["rank"] == 1 + assert semantic_hit["retrieval_provenance"]["vector"]["distance"] == 0.0 + assert diagnostics["vector_lane"]["status"] == "active" + assert diagnostics["vector_lane"]["allowed_vector_id_count"] == 2 + finally: + await graph.close() + await vector.close() + await sql.close() diff --git a/tests/unit/test_phase7_qualification_fixtures.py b/tests/unit/test_phase7_qualification_fixtures.py index 6662abe..37bf149 100644 --- a/tests/unit/test_phase7_qualification_fixtures.py +++ b/tests/unit/test_phase7_qualification_fixtures.py @@ -191,7 +191,9 @@ def test_11_fixture_id_absent_from_identity_map(baseline_setup) -> None: fid = s_auth["case_evidence_fixtures"]["cross_tenant_search"][0] # Remove row from identity map filtered_rows = [r for r in rows if r.get("evidence_id") != fid] - with pytest.raises(QualificationFixtureError, match="is absent from frozen identity authority|unknown frozen identity"): + with pytest.raises( + QualificationFixtureError, match="is absent from frozen identity authority|unknown frozen identity" + ): validate_qualification_fixtures( qualification_scope=q_scope, scope_test_authority=s_auth, @@ -412,7 +414,11 @@ def test_publisher_compatibility_for_qualification_fixtures(tmp_path: Path) -> N ver_id = row["version_id"] matching_articles = [ - r for r in records if r["record_type"] == "article" and r["legislation_id"] == doc_id and r["legislation_version_id"] == ver_id + r + for r in records + if r["record_type"] == "article" + and r["legislation_id"] == doc_id + and r["legislation_version_id"] == ver_id ] assert len(matching_articles) >= 1, f"Missing article record for fixture {fixture_id}" art = matching_articles[0] @@ -440,4 +446,3 @@ def test_publisher_compatibility_for_qualification_fixtures(tmp_path: Path) -> N ) assert idemp.startswith("mesa-data:") assert len(idemp) == 74 - diff --git a/tests/unit/test_publisher_chunker.py b/tests/unit/test_publisher_chunker.py index 21724e4..2d97c52 100644 --- a/tests/unit/test_publisher_chunker.py +++ b/tests/unit/test_publisher_chunker.py @@ -121,3 +121,19 @@ def test_chunker_deterministic_invariants(): assert c1.content == c2.content assert c1.char_start == c2.char_start assert c1.char_end == c2.char_end + + +def test_chunker_caps_transport_chunks_to_mesa_evidence_span_contract(): + canonical_text = "A" * 4500 + + chunks = plan_source_chunks( + document_id="doc-vector-contract", + version_id="doc-vector-contract:v1", + canonical_text=canonical_text, + records=[], + content_limit_chars=32768, + ) + + assert len(chunks) == 2 + assert max(len(chunk.content) for chunk in chunks) <= 4096 + assert "".join(chunk.content for chunk in chunks) == canonical_text diff --git a/tests/unit/test_publisher_client_and_ledger.py b/tests/unit/test_publisher_client_and_ledger.py index 64d055a..b1bdc4d 100644 --- a/tests/unit/test_publisher_client_and_ledger.py +++ b/tests/unit/test_publisher_client_and_ledger.py @@ -133,7 +133,7 @@ def assert_insert(request): "title": "Madde 3", "source_ref": "https://authority.test/doc-1", "content": "canonical text", - "evidence_span": "", + "evidence_span": "canonical text", "revision_number": 2, "chunk_ordinal": 3, "finalize_revision": True, @@ -159,6 +159,29 @@ def assert_insert(request): assert start.called and insert.called +def test_native_v4_payload_rejects_chunks_larger_than_evidence_contract(): + client = MesaClient(MesaTargetSettings(), api_key="secret") + chunk = SourceChunk( + chunk_id="v1:chunk:oversized", + document_id="doc-1", + version_id="v1", + chunk_type="general", + char_start=0, + char_end=4097, + ordinal=1, + content="A" * 4097, + content_hash="content-hash", + ) + + with pytest.raises(ValueError, match="evidence span contract"): + client.build_memory_insert_payload( + chunk, + session_id="sess-1", + idempotency_key="stable-key", + finalize_revision=True, + ) + + @pytest.mark.parametrize( ("raw", "expected"), [ diff --git a/uv.lock b/uv.lock index 660e87b..830711a 100644 --- a/uv.lock +++ b/uv.lock @@ -1410,11 +1410,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]]