diff --git a/.mesa_data_agent_state.json b/.mesa_data_agent_state.json deleted file mode 100644 index a801a20..0000000 --- a/.mesa_data_agent_state.json +++ /dev/null @@ -1,546 +0,0 @@ -{ - "schema_version": 1, - "project": "MESA_Data", - "workflow": "mesa_data_autonomy", - "branch": "fix/mesa-data-final-mvp", - "baseline_commit": "84c61db716c10cb4110b673f9a976215afe71cd3", - "current_phase": "final_p1_record_instance_not_null", - "overall_status": "ready", - "phases": { - "prompt_1": { - "status": "completed", - "started_at": "2026-08-29T01:42:00Z", - "completed_at": "2026-08-29T01:54:00Z", - "commits": [ - "5014d5d fix: make document versions deterministic and isolate record instances", - "964b72f feat: add canonical source spans, parser coverage, and text health checks", - "bedcb99 feat: add deterministic 3-state quality gate and release guard", - "d6efb7c test: cover version, parser, quality gate, and migration integrity" - ], - "files_changed": [ - "migrations/0005_version_aware_records_and_quality.sql", - "src/mesa_legal_data/catalog.py", - "src/mesa_legal_data/pipeline.py", - "src/mesa_legal_data/parsers/text_normalizer.py", - "src/mesa_legal_data/parsers/legislation.py", - "src/mesa_legal_data/parsers/encoding.py", - "src/mesa_legal_data/parsers/coverage.py", - "src/mesa_legal_data/quality.py", - "src/mesa_legal_data/web/api.py", - "tests/unit/test_version_integrity.py", - "tests/unit/test_canonical_spans_and_coverage.py", - "tests/unit/test_quality_gate.py", - "tests/unit/test_migration_0005.py" - ], - "migrations": [ - "0005_version_aware_records_and_quality.sql" - ], - "findings": [ - "Version ID in pipeline was using execution timestamp instead of source artifact publication/snapshot/retrieved date, creating fake duplicate versions on daily reprocessing.", - "Records table used record_id as PRIMARY KEY without version scoping, causing Version B to overwrite Version A's record instances for the same logical article.", - "Version kind calculation was hardcoded to 'consolidated_snapshot' when inserted into catalog.", - "ParsedArticle had no char_start, char_end, or ordinal, and pipeline did not compute canonical coordinate spans.", - "No formal interval-merge coverage computation existed to detect preamble/annex uncovered ranges.", - "No formal 3-state Quality Gate (PASS/REVIEW/BLOCK) existed at version level to guard releases against corrupt extractions/spans." - ], - "decisions": [ - "Upgraded records table to use composite uniqueness (version_id, record_id) and record_instance_id primary key to isolate multi-version record instances.", - "Upgraded versions schema to track revision_number sequentially per document, supersedes_version_id, and quality_status/quality_json.", - "Implemented single coordinate system flow: RAW -> EXTRACTED -> SAFE NORMALIZATION -> CANONICAL TEXT -> LEGAL PARSER -> SOURCE SPANS.", - "Implemented interval-merge coverage computation in coverage.py to classify uncovered ranges (preamble, gap, annex_trailing).", - "Implemented 3-state Quality Gate evaluating 9 check groups (SOURCE, RAW, EXTRACTION, STRUCTURE, METADATA, CITATION, PRIVACY, PROVENANCE, DUPLICATE) returning strictly PASS, REVIEW, or BLOCK.", - "Implemented Release Guard in catalog and release iterator preventing approval or release of versions with quality == BLOCK." - ], - "tests": [ - "tests/unit/test_version_integrity.py", - "tests/unit/test_canonical_spans_and_coverage.py", - "tests/unit/test_quality_gate.py", - "tests/unit/test_migration_0005.py", - "tests/unit/test_encoding_cp1254.py" - ], - "blockers": [] - }, - "prompt_2": { - "status": "completed", - "started_at": "2026-08-29T02:00:00Z", - "completed_at": "2026-08-29T02:08:00Z", - "commits": [ - "390f6ecdc002d351bb356737799e2b2d0b888558 feat: citation semantics, safe auto-approval engine, and web panel completion" - ], - "files_changed": [ - "migrations/0006_citations_certifications_and_panel.sql", - "schemas/citation.schema.json", - "src/mesa_legal_data/ids.py", - "src/mesa_legal_data/parsers/citations.py", - "src/mesa_legal_data/catalog.py", - "src/mesa_legal_data/pipeline.py", - "src/mesa_legal_data/web/schemas.py", - "src/mesa_legal_data/web/api.py", - "src/mesa_legal_data/web/static/index.html", - "src/mesa_legal_data/web/static/app.js", - "tests/unit/test_citation_semantics.py", - "tests/unit/test_auto_approval_and_certifications.py", - "tests/integration/test_web_panel_prompt2.py" - ], - "migrations": [ - "0006_citations_certifications_and_panel.sql" - ], - "findings": [ - "Extracted citations were previously automatically treated as valid, violating the 4-state lifecycle (EXTRACTED, RESOLVED, UNRESOLVED, HUMAN_VERIFIED).", - "Citation spans lacked exact unicode coordinate matching canonical text slices.", - "Auto-approval lacked strict certification verification and configurable audit quality sampling.", - "Web panel needed 6 key metrics, truthful local staging publish labels, multi-version document history, reprocess actions, and source automation settings." - ], - "decisions": [ - "Implemented 4-state citation lifecycle with Turkish legal aliases (TCK, CMK, HMK, TMK, TBK, TTK, \u0130\u0130K, \u0130YUK, VUK, KVKK, AY, AvK, PVSK, \u0130\u015fK) and temporal relation verbs (amends, repeals, adds).", - "Enforced exact canonical coordinate slicing (canonical_text[char_start:char_end] == raw_text) and context-aware law number matching.", - "Implemented 7-condition Safe Auto-Approval Engine in catalog.py with parser certifications and weekly audit sampling.", - "Completed web management panel with health metrics, honest MESA banner, multi-version history, safe reprocessing, and operational settings." - ], - "tests": [ - "tests/unit/test_citation_semantics.py", - "tests/unit/test_auto_approval_and_certifications.py", - "tests/integration/test_web_panel_prompt2.py" - ], - "blockers": [] - }, - "prompt_3": { - "status": "completed", - "started_at": "2026-08-29T02:08:00Z", - "completed_at": "2026-08-29T02:17:00Z", - "commits": [ - "59a822d3b5d1187fec4683c71ad1da831745b318 feat: implement real MESA v4 publisher, legal chunker, delivery ledger, cross-release deduplication, and web push panel" - ], - "files_changed": [ - "migrations/0007_mesa_v4_publisher_and_deliveries.sql", - "src/mesa_legal_data/publisher/__init__.py", - "src/mesa_legal_data/publisher/models.py", - "src/mesa_legal_data/publisher/hashing.py", - "src/mesa_legal_data/publisher/chunker.py", - "src/mesa_legal_data/publisher/client.py", - "src/mesa_legal_data/publisher/ledger.py", - "src/mesa_legal_data/publisher/engine.py", - "src/mesa_legal_data/operations.py", - "src/mesa_legal_data/web/schemas.py", - "src/mesa_legal_data/web/api.py", - "src/mesa_legal_data/web/static/index.html", - "src/mesa_legal_data/web/static/app.js", - "src/mesa_legal_data/cli.py", - "tests/unit/test_publisher_chunker.py", - "tests/unit/test_publisher_idempotency_and_hashing.py", - "tests/unit/test_publisher_client_and_ledger.py", - "tests/integration/test_mesa_v4_publisher_e2e.py", - "tests/integration/test_web_panel_publisher.py", - "tests/unit/test_cli_sync.py" - ], - "migrations": [ - "0007_mesa_v4_publisher_and_deliveries.sql" - ], - "findings": [ - "MESA Publisher was missing; MESA_Data had no direct v4 HTTP client, preflight checklist, or cross-release deduplication ledger.", - "Legal chunking requires preserving preamble, article units, and annex schedules with deterministic fallback on paragraph boundaries.", - "API key secrecy required strict separation from SQLite persistence, logs, JSON state, and browser source code.", - "MESA mutation state tracking requires truthful reporting (only COMMITTED marks success, partial failures enable targeted retry).", - "Automated pipeline sync (mesa-data sync) must stop before MESA push to preserve human-in-the-loop manual push gate." - ], - "decisions": [ - "Target settings: Non-secret target settings persisted in SQLite table mesa_target_settings, secret API key read strictly from MESA_DATA_MESA_API_KEY / MESA_API_KEY environment variable.", - "Legal chunker: Preserves preamble, articles, and annex ranges with configurable content limit (default 32768) and deterministic paragraph boundaries (\\n\\n).", - "Deterministic hashing: Canonical revision hash is pure SHA256 of sorted chunk content hashes. Stable idempotency key mesa-data: contains zero runtime volatile data.", - "Delivery ledger & O(1) deduplication: Persists mesa_deliveries and mesa_delivery_items; skips already COMMITTED chunks across releases.", - "Truthful mutation lifecycle: COMMITTED, PROCESSING, QUEUED, FAILED, REJECTED tracked accurately; partial failure banner displays retry CTA.", - "Web Panel Yay\u0131nla tab: Target settings manager, preflight checklist, ready summary, confirmation modal, live progress polling, delivery detail modal, retry CTA.", - "CLI sync: mesa-data sync command discovers, fetches, processes, validates quality, auto-approves, and summarizes ready items, stopping strictly before MESA push." - ], - "tests": [ - "tests/unit/test_publisher_chunker.py", - "tests/unit/test_publisher_idempotency_and_hashing.py", - "tests/unit/test_publisher_client_and_ledger.py", - "tests/integration/test_mesa_v4_publisher_e2e.py", - "tests/integration/test_web_panel_publisher.py", - "tests/unit/test_cli_sync.py" - ], - "blockers": [] - }, - "prompt_4": { - "status": "completed", - "started_at": "2026-08-29T02:18:00Z", - "completed_at": "2026-08-29T02:24:00Z", - "commits": [ - "7f958f0ac6592f8e0f6bd2c4697a15ca626993b4 test: integration audit and verification suite for Prompt 4 Kontrol 1-11" - ], - "files_changed": [ - "tests/integration/test_prompt4_integration_audit.py", - ".mesa_data_agent_state.json", - "docs/BUILD_STATE.json" - ], - "migrations": [], - "findings": [ - "All 11 audit controls verified against source code, SQLite schemas, HTTP APIs, and runtime behavior.", - "No live MESA endpoint is currently deployed on localhost/network in this workspace; verified strictly through mock HTTP contracts; marked contract state as LIVE_MESA_NOT_VERIFIED.", - "Verified strict manual push gate: CLI sync never performs automated MESA push.", - "Zero prohibited overengineered components found (no Celery, Kafka, RabbitMQ, Airflow, Spark, LLMs, Vector DBs, Graph DBs, or frontend frameworks)." - ], - "decisions": [ - "Created comprehensive Prompt 4 integration audit test suite covering Kontrol 1 through 11 in tests/integration/test_prompt4_integration_audit.py.", - "Verified all 11 core invariants end-to-end with 295 passing tests (291 non-scale, 4 scale, 0 failed)." - ], - "tests": [ - "tests/integration/test_prompt4_integration_audit.py", - "Full test suite: 295 tests passed (291 non-scale + 4 scale, 0 failed, 100% pass rate)" - ], - "blockers": [] - }, - "correctness_hardening": { - "status": "completed", - "started_at": "2026-08-29T00:00:00Z", - "completed_at": "2026-08-29T00:00:00Z", - "findings": [ - { - "id": "QG_ZERO_ARTICLE_PASS", - "severity": "P0", - "status": "fixed" - }, - { - "id": "PUBLICATION_DATE_BRIDGE_LOSS", - "severity": "P0", - "status": "fixed" - }, - { - "id": "STALE_APPROVED_VERSION_PUBLISH", - "severity": "P0", - "status": "fixed" - }, - { - "id": "CURRENT_VERSION_MESA_STATUS", - "severity": "P0", - "status": "fixed" - }, - { - "id": "DERIVED_REPROCESS_EARLY_RETURN", - "severity": "P0", - "status": "fixed" - }, - { - "id": "MESA_TARGET_KEY_EXFILTRATION", - "severity": "P0", - "status": "fixed" - }, - { - "id": "MUTATION_TIMEOUT_MARKED_FAILED", - "severity": "P1", - "status": "fixed" - }, - { - "id": "ARTIFACT_SHA_CROSS_DOCUMENT_COLLISION", - "severity": "P1", - "status": "fixed" - } - ], - "fixes": [ - "Legislation zero-article and abnormal large uncovered-area outcomes cannot PASS; article ordinal is retained in canonical records.", - "Discovery publication_date is forwarded into artifact metadata and canonical provenance; retrieval time remains fallback-only.", - "Release and MESA selection now joins documents.current_version_id, excluding stale approved versions and unrelated blockers.", - "Reprocess refreshes only derived output, resets approval, and preserves version identity and revision number.", - "MESA requests require local development host or HTTPS host exactly allowed by MESA_DATA_MESA_ALLOWED_HOST; 401/403 are reachable but unauthenticated failures.", - "Pending mutation timeouts use AWAITING_MUTATION and schema support; retries poll known mutation IDs before resubmitting.", - "Same payload for another document produces an explicit ARTIFACT_DOCUMENT_COLLISION rather than reassociation." - ], - "commits": [ - "a6656bb fix: harden parsing and version correctness", - "c20aa17 fix: secure current-version MESA publishing", - "71f4306 fix: report MESA state from current version" - ], - "migrations": [ - "0009_correctness_hardening.sql" - ], - "tests": [ - { - "name": "focused correctness regression", - "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_quality_gate.py tests/unit/test_catalog_release_iterator.py tests/unit/test_publisher_client_and_ledger.py tests/integration/test_mesa_v4_publisher_e2e.py", - "result": "19 passed" - }, - { - "name": "compile", - "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run python -m compileall -q src/mesa_legal_data", - "result": "passed" - }, - { - "name": "ruff", - "command": "UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run ruff check src tests", - "result": "passed" - }, - { - "name": "current-version MESA status regression", - "command": "MESA_DATA_MESA_ALLOWED_HOST=mock-mesa.internal UV_CACHE_DIR=/tmp/mesa-data-uv-cache uv run pytest -q tests/unit/test_correctness_hardening.py tests/unit/test_publisher_client_and_ledger.py", - "result": "10 passed" - } - ], - "blockers": [ - "The all-test pytest invocation did not complete within the desktop command window and is not recorded as passed." - ], - "unverified": [ - "MESA live HTTP payload and server-side mutation contract remain UNVERIFIED because no live endpoint or MESA source code is available." - ] - }, - "prompt_5": { - "status": "completed", - "completed_at": "2026-08-29T15:13:48Z", - "commits": [ - "1baa2c2642684a24aead899c3d53873c60298342 fix: harden final autonomy audit invariants" - ], - "migrations": [ - "0008_final_audit_hardening.sql" - ], - "findings": [ - "Publisher routes were guessed rather than documented or explicitly configured, and unknown remote states could be misreported as COMMITTED.", - "Publisher payload assembly mixed citation text into canonical document text and could release multiple approved historical versions of one document.", - "Auto-approval trusted caller-supplied parser/source/quality facts instead of the stored version and artifact facts.", - "Exact artifact reprocessing could rewrite immutable version pointers, and predecessor links were inferred without authoritative evidence.", - "Quality checks accepted shifted spans and did not reliably block severe replacement loss or missing middle sections.", - "Numbered but unregistered legislation citations were incorrectly promoted directly to RESOLVED.", - "The sync item limit was not a total-run bound and could process more backlog items than requested.", - "Browser execution found a collect-view JavaScript scope regression not covered by the prior static UI tests.", - "No live MESA endpoint or MESA source code was available; the remote contract and server-side deduplication remain unverified." - ], - "decisions": [ - "Removed all guessed MESA routes and required an explicit configured contract before connectivity or publication.", - "Made COMMITTED fail closed: missing, accepted, received, queued, processing, timeout, rejected, and unknown states cannot become success.", - "Bound approvals, releases, canonical reads, and reprocessing to exact stored version/artifact identities.", - "Hardened deterministic quality, citation lifecycle, retry/idempotency, partial delivery, and total sync-limit behavior.", - "Verified the primary panel journey in a real local browser and added a regression guard for collect-view helper scope." - ], - "tests": [ - "uv run pytest -ra: 304 passed in 71.01s", - "uv run ruff check src tests: passed", - "uv run mypy src/mesa_legal_data --ignore-missing-imports --explicit-package-bases --follow-imports=skip: passed (77 source files)", - "python -m compileall -q src/mesa_legal_data: passed", - "node --check src/mesa_legal_data/web/static/app.js: passed", - "Live browser journey: Ana Sayfa, Veri Topla, K\u00fct\u00fcphane, \u0130nceleme, Yay\u0131nla, advanced navigation and explicit MESA contract fields verified", - "Two bounded real Resmi Gazete sync runs in isolated storage: one item per run, zero MESA deliveries" - ], - "blockers": [ - "Live MESA HTTP contract and server-side idempotency could not be verified because no endpoint or MESA source access was available." - ] - }, - "master_mvp_closure": { - "status": "completed", - "started_at": "2026-08-29T21:00:00Z", - "completed_at": "2026-08-29T22:35:00Z", - "commits": [ - "feat: master MVP closure, legal chronology current version recomputation, instance-scoped reviews, release-bound publisher delivery, and 100% green CI parity" - ], - "files_changed": [ - "migrations/0010_mvp_master_closure.sql", - "src/mesa_legal_data/catalog.py", - "src/mesa_legal_data/pipeline.py", - "src/mesa_legal_data/quality.py", - "src/mesa_legal_data/ids.py", - "src/mesa_legal_data/sources/manual.py", - "src/mesa_legal_data/harvest/service_bridge.py", - "src/mesa_legal_data/harvest/queue.py", - "src/mesa_legal_data/release/builder.py", - "src/mesa_legal_data/publisher/engine.py", - "src/mesa_legal_data/operations.py", - "src/mesa_legal_data/web/api.py", - "tests/integration/test_master_mvp_closure.py", - "tests/unit/test_ids.py", - "tests/unit/test_version_integrity.py", - "tests/acceptance/test_web_user_journey_contract.py", - "pyproject.toml", - "uv.lock", - ".github/workflows/ci.yml" - ], - "migrations": [ - "0010_mvp_master_closure.sql" - ], - "findings": [ - "Multi-version legal chronology needed automatic recomputation when historical versions backfill.", - "Single-record reviews and bulk approvals must scope strictly to record_instance_id (version_id:record_id) to avoid cross-version mutation.", - "Release builder must scope blocker issues to candidate versions and record instances rather than blocking on unrelated catalogue issues.", - "Publisher delivery plan must support building strictly from verified release items and canonicals.", - "Publisher cancellation callback must halt sending immediately and report truthful CANCELLED delivery status.", - "Missing publication date on original publications must flag PUBLICATION_DATE_MISSING and set quality to REVIEW.", - "Duplicate artifact detection must truthfully set REVIEW in quality check." - ], - "decisions": [ - "Implemented migration 0010 adding unique non-null review_id and record_instance_id in record_reviews, version_id/record_instance_id in validation_issues, and partial/awaiting_external operation job statuses.", - "Implemented legal date normalization and recompute_document_current_version in catalog.py and pipeline.py.", - "Bounded single-record reviews and bulk streaming approvals strictly to record_instance_id with unique review IDs.", - "Made release builder blocker checks candidate-scoped via spool_conn version/record IDs.", - "Supported release_id verification and item binding in publisher delivery engine with immediate cancellation handling.", - "Enforced 16-hex version ID hash and full canonical text SHA-256.", - "Verified all 19 master test suite cases A through AG and guaranteed 100% clean CI (ruff, mypy, pip-audit, pytest)." - ], - "tests": [ - "uv run pytest -m 'not scale' -ra: 322 passed", - "uv run pytest -m 'scale' -ra: 4 passed", - "uv run ruff format --check .: passed (210 files)", - "uv run ruff check .: passed", - "uv run mypy src: passed (77 source files)", - "uv run pip-audit: passed (no known vulnerabilities)" - ], - "blockers": [ - "Live MESA HTTP contract and server-side idempotency could not be verified because no endpoint or MESA source access was available." - ] - }, - "post_master_verification": { - "status": "completed", - "started_at": "2026-08-29T22:57:47Z", - "completed_at": "2026-08-29T23:15:00Z", - "verified_invariants": [ - "Control 1-6: Document version chronology, legal date order (2026-05-10 > 2025-08-01 > 2024-03-12), unknown date safety, and stable resolution verified", - "Control 7-10: Historical approval/rejection/fetch lifecycle isolation, document current version protection, and revision number immutability verified", - "Control 11-17: Parser certified auto-approval, single-pass pipeline streaming, coordinate spans [start, end], and interval-merge coverage verified", - "Control 18-20: Cross-version record review isolation, 409 ambiguity guard on unqualified record IDs, and unique non-null review IDs verified", - "Control 21-25: Version/document scoped validation issues, candidate blocker release isolation, and privacy zero-leak verified", - "Control 26-30: Citation target resolution, multi-version document targeting, and cascade deletion protection verified", - "Control 31-35: 3-state Quality Gate (PASS/REVIEW/BLOCK), zero-article/huge preamble blocking, and release blocking on non-PASS verified", - "Control 36-40: Published release immutability, SHA256 manifest trust anchor, TOCTOU frozen release publisher isolation verified", - "Control 41-43: Seed collector, Resmi Gazete pilot genuine extraction, and harvest budget/throttle controls verified", - "Control 44-53: MESA v4 publisher client, allowed host safety, idempotency hashing, deduplication, partial state handling, and retry recovery verified", - "Control 54-56: Web admin UI CSRF/Origin enforcement, pipeline gate bypass prevention, and live secret protection verified", - "Control 57-60: Fresh & upgrade database migrations 0001-0010, foreign key pragma, integrity pragma, and legacy backfill verified", - "Control 61-65: CI/CD gate conformity, lockfile sync, static types, formatting, linting, and pip-audit 0 vulnerabilities verified" - ], - "failed_invariants": [], - "fixes": [], - "tests": [ - "uv run pytest -m 'not scale' -ra: 347 passed", - "uv run pytest -m 'scale' -ra: 4 passed", - "uv run pytest tests/integration/test_post_master_independent_audit.py: 25 passed" - ], - "ci_results": [ - "uv sync --frozen: passed", - "uv run ruff format --check .: passed (211 files)", - "uv run ruff check .: passed", - "uv run mypy src: passed (77 source files)", - "uv run pip-audit: passed (0 known vulnerabilities)" - ], - "remaining_risks": [ - "Live MESA HTTP endpoint remains UNVERIFIED because external network access/live MESA instance is outside repository boundary." - ] - }, - "final_mvp_blocker_closure": { - "status": "completed", - "started_at": "2026-08-30T00:00:00+03:00", - "completed_at": "2026-08-30T00:43:56+03:00", - "commits": [ - "d6a35b7 fix: close final MVP integrity blockers" - ], - "migrations": [ - "0011_final_mvp_blocker_closure.sql" - ], - "findings": [ - "Panel confirmation was bound to a mutable live-catalog summary rather than an exact immutable release and target fingerprint.", - "Publisher release reconstruction could join identical logical record IDs and hashes across legal versions without exact version identity.", - "Ambiguous legacy reviews and logical-record issues could contaminate another version.", - "Single-record revision mutation was scoped by logical record_id and could alter sibling versions.", - "String-shaped but impossible calendar dates could enter chronology and approval decisions.", - "Identical artifact bytes could not safely enrich missing authoritative publication metadata.", - "Release payload normalization required separate canonical-record and packaged-payload hashes." - ], - "tests": [ - "fresh and populated-upgrade migration integrity: passed in test_control_57_and_58_fresh_and_upgrade_database_integrity", - "uv sync --frozen: passed (79 packages audited)", - "uv run ruff format --check .: passed (211 files)", - "uv run ruff check .: passed", - "uv run mypy src: passed (77 source files)", - "uv run pip-audit: passed (0 known vulnerabilities)", - "uv run pytest -m 'not scale' -ra --durations=20: 350 passed, 0 failed, 4 deselected", - "uv run pytest -m 'scale' -ra --durations=20: 4 passed, 0 failed, 350 deselected" - ], - "blockers": [], - "remaining_risks": [ - "Live MESA HTTP request/response and server-side idempotency semantics remain unverified." - ] - }, - "final_p1_record_instance_not_null": { - "status": "completed", - "started_at": "2026-08-30T23:20:00+03:00", - "completed_at": "2026-08-30T23:35:00+03:00", - "commits": [ - "fix: enforce non-null record instance identities" - ], - "migrations": [ - "0012_record_instance_not_null.sql" - ], - "files_changed": [ - "migrations/0012_record_instance_not_null.sql", - "tests/unit/test_migration_0012.py", - "tests/integration/test_post_master_independent_audit.py", - "tests/acceptance/test_workspace_e2e.py", - "tests/acceptance/test_operations_scale_gate.py", - "tests/integration/test_annotations_and_issues.py", - "tests/integration/test_data_explorer_api.py", - "tests/integration/test_large_export_streaming.py", - "tests/integration/test_record_revisions.py", - "tests/unit/test_final_closure_gate.py", - "tests/unit/test_release.py", - "docs/BUILD_STATE.json" - ], - "findings": [ - "records.record_instance_id was TEXT PRIMARY KEY without explicit NOT NULL, allowing SQLite to accept NULL values on insert.", - "Deterministic authoritative generator version_id:record_id backfills any legacy NULL record_instance_id values." - ], - "decisions": [ - "Added migration 0012_record_instance_not_null.sql performing safe table rebuild to declare record_instance_id TEXT NOT NULL PRIMARY KEY.", - "Preserved (version_id, record_id) UNIQUE constraint, foreign keys, and indexes (idx_records_version_record, idx_records_record_id, idx_records_version_type, idx_records_approval_status).", - "Preserved and restored triggers on release_items referencing records." - ], - "tests": [ - "Test A (Fresh DB): passed in test_migration_0012.py::test_migration_0012_test_a_fresh_db_table_info", - "Test B (NULL insert rejected): passed in test_migration_0012.py::test_migration_0012_test_b_null_insert_rejected", - "Test C (Normal insert works): passed in test_migration_0012.py::test_migration_0012_test_c_normal_insert_works", - "Test D (Upgrade populated DB): passed in test_migration_0012.py::test_migration_0012_test_d_upgrade_populated_db", - "Test E (Legacy NULL backfill): passed in test_migration_0012.py::test_migration_0012_test_e_legacy_null_backfill", - "Test F (Version-aware invariant): passed in test_migration_0012.py::test_migration_0012_test_f_version_aware_invariant", - "uv sync --frozen: passed", - "uv run ruff format --check .: passed (212 files)", - "uv run ruff check .: passed", - "uv run mypy src: passed (77 source files)", - "uv run pip-audit: passed (0 known vulnerabilities)", - "uv run pytest -m \"not scale\" -ra --durations=20: 356 passed, 0 failed, 4 deselected", - "uv run pytest -m \"scale\" -ra --durations=20: 4 passed, 0 failed, 356 deselected", - "uv run pytest -ra: 360 passed in 72.83s" - ], - "blockers": [] - } - }, - "mesa_contract": { - "status": "unverified", - "source": "explicit_configuration_and_local_mock_contract", - "verified_by_live_http": false, - "assumptions": [], - "unknowns": [ - "Live MESA health, publish, and mutation-status paths", - "Live MESA request and response schemas", - "Live MESA asynchronous mutation state semantics", - "Live MESA server-side idempotency and duplicate suppression behavior" - ], - "policy": "Unknown routes are never guessed; all three routes must be explicitly configured from a documented or live-verified contract." - }, - "invariants": { - "immutable_artifacts": true, - "version_record_isolation": true, - "deterministic_canonical_spans": true, - "quality_gate_release_guard": true, - "citation_lifecycle_enforced": true, - "certified_parser_auto_approval": true, - "mesa_v4_publisher_ledger": true, - "mesa_cross_release_dedup": true, - "mesa_api_key_secret_isolation": true, - "manual_push_gate_enforced": true, - "explicit_mesa_contract_required": true, - "mesa_live_contract_verified": false - }, - "handoff": { - "next_phase": null, - "summary": "Final P1 blocker fix complete. records.record_instance_id is strictly NOT NULL PRIMARY KEY at database level via migration 0012. All 360 tests pass (356 non-scale + 4 scale), fresh and upgrade migrations pass, CI gates green, 0 P0/P1 blockers remain.", - "must_verify": [ - "Ensure live deployment environment configures MESA_DATA_MESA_API_KEY securely.", - "Configure documented MESA routes and verify health, publish, polling, COMMITTED truth, and server-side idempotency against a live endpoint." - ] - }, - "last_updated_at": "2026-08-30T23:35:00+03:00" -} diff --git a/config/harvest.yaml b/config/harvest.yaml index 76dad00..c6770cd 100644 --- a/config/harvest.yaml +++ b/config/harvest.yaml @@ -38,6 +38,7 @@ sources: - "CUMHURBAŞKANI KARARLARI" - "YÖNETMELİKLER" - "TEBLİĞLER" + - "ANAYASA MAHKEMESİ KARARLARI" exclude_sections: - "İLÂN BÖLÜMÜ" @@ -50,6 +51,7 @@ sources: - presidential_decision - regulation - communique + - court_decision budget: target_raw_bytes: 5368709120 diff --git a/config/seed_legislation.yaml b/config/seed_legislation.yaml index 30c25bc..0e05efc 100644 --- a/config/seed_legislation.yaml +++ b/config/seed_legislation.yaml @@ -27,7 +27,7 @@ seed_legislation: document_id: "tr:legislation:law:2004" title: "İcra ve İflas Kanunu" type: "law" - source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.2004.pdf" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.2004.pdf" - number: "5237" document_id: "tr:legislation:law:5237" @@ -70,3 +70,177 @@ seed_legislation: title: "Kişisel Verilerin Korunması Kanunu" type: "law" source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6698.pdf" + + - number: "6502" + document_id: "tr:legislation:law:6502" + title: "Tüketicinin Korunması Hakkında Kanun" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6502.pdf" + + - number: "5326" + document_id: "tr:legislation:law:5326" + title: "Kabahatler Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5326.pdf" + + - number: "6325" + document_id: "tr:legislation:law:6325" + title: "Hukuk Uyuşmazlıklarında Arabuluculuk Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6325.pdf" + + - number: "7036" + document_id: "tr:legislation:law:7036" + title: "İş Mahkemeleri Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.7036.pdf" + + - number: "7201" + document_id: "tr:legislation:law:7201" + title: "Tebligat Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.7201.pdf" + + - number: "1512" + document_id: "tr:legislation:law:1512" + title: "Noterlik Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.1512.pdf" + + - number: "492" + document_id: "tr:legislation:law:492" + title: "Harçlar Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.492.pdf" + + - number: "213" + document_id: "tr:legislation:law:213" + title: "Vergi Usul Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.213.pdf" + + - number: "193" + document_id: "tr:legislation:law:193" + title: "Gelir Vergisi Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.193.pdf" + + - number: "3065" + document_id: "tr:legislation:law:3065" + title: "Katma Değer Vergisi Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.3065.pdf" + + - number: "6183" + document_id: "tr:legislation:law:6183" + title: "Amme Alacaklarının Tahsil Usulü Hakkında Kanun" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.6183.pdf" + + - number: "634" + document_id: "tr:legislation:law:634" + title: "Kat Mülkiyeti Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.634.pdf" + + - number: "2644" + document_id: "tr:legislation:law:2644" + title: "Tapu Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.2644.pdf" + + - number: "3194" + document_id: "tr:legislation:law:3194" + title: "İmar Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.3194.pdf" + + - number: "2942" + document_id: "tr:legislation:law:2942" + title: "Kamulaştırma Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.2942.pdf" + + - number: "4734" + document_id: "tr:legislation:law:4734" + title: "Kamu İhale Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.4734.pdf" + + - number: "5018" + document_id: "tr:legislation:law:5018" + title: "Kamu Malî Yönetimi ve Kontrol Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5018.pdf" + + - number: "657" + document_id: "tr:legislation:law:657" + title: "Devlet Memurları Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.657.pdf" + + - number: "2547" + document_id: "tr:legislation:law:2547" + title: "Yükseköğretim Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.2547.pdf" + + - number: "5510" + document_id: "tr:legislation:law:5510" + title: "Sosyal Sigortalar ve Genel Sağlık Sigortası Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5510.pdf" + + - number: "5846" + document_id: "tr:legislation:law:5846" + title: "Fikir ve Sanat Eserleri Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.3.5846.pdf" + + - number: "6769" + document_id: "tr:legislation:law:6769" + title: "Sınai Mülkiyet Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6769.pdf" + + - number: "4054" + document_id: "tr:legislation:law:4054" + title: "Rekabetin Korunması Hakkında Kanun" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.4054.pdf" + + - number: "5411" + document_id: "tr:legislation:law:5411" + title: "Bankacılık Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5411.pdf" + + - number: "6362" + document_id: "tr:legislation:law:6362" + title: "Sermaye Piyasası Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6362.pdf" + + - number: "5941" + document_id: "tr:legislation:law:5941" + title: "Çek Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5941.pdf" + + - number: "6563" + document_id: "tr:legislation:law:6563" + title: "Elektronik Ticaretin Düzenlenmesi Hakkında Kanun" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6563.pdf" + + - number: "5651" + document_id: "tr:legislation:law:5651" + title: "İnternet Ortamında Yapılan Yayınların Düzenlenmesi ve Bu Yayınlar Yoluyla İşlenen Suçlarla Mücadele Edilmesi Hakkında Kanun" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5651.pdf" + + - number: "5275" + document_id: "tr:legislation:law:5275" + title: "Ceza ve Güvenlik Tedbirlerinin İnfazı Hakkında Kanun" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.5275.pdf" diff --git a/config/settings.yaml b/config/settings.yaml deleted file mode 100644 index edbf5fa..0000000 --- a/config/settings.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: "1.0.0" - -settings: - data_root: "/storage/mesa-legal-data/data" - mesa_staging_db: "/storage/mesa-legal-data/data/mesa_staging.sqlite" - environment: "development" - log_level: "INFO" - operator_contact: "test@example.com" - http_proxy: null - - storage: - active_data_limit_gb: 50.0 - raw_limit_gb: 25.0 - canonical_limit_gb: 10.0 - releases_limit_gb: 10.0 - tmp_limit_gb: 5.0 - minimum_free_space_gb: 20.0 diff --git a/config/sources.yaml b/config/sources.yaml index 3117190..fbbc7ec 100644 --- a/config/sources.yaml +++ b/config/sources.yaml @@ -7,7 +7,7 @@ sources: base_url: "https://www.resmigazete.gov.tr/" access_mode: "approved_web" enabled: true - families: ["legislation"] + families: ["legislation", "decision"] source_role: "original_publication" policy_version: "1.0.0" http: @@ -40,7 +40,7 @@ sources: min_interval_seconds: 5 timeout_seconds: 30 retries: 3 - max_requests_per_run: 25 + max_requests_per_run: 60 max_download_bytes: 52428800 allowed_content_types: - "application/pdf" @@ -76,17 +76,20 @@ sources: authority: "T.C. Yargıtay Başkanlığı" base_url: "https://karararama.yargitay.gov.tr/" access_mode: "manual" - enabled: false + enabled: true families: ["decision"] source_role: "official_case_law" policy_version: "1.0.0" http: user_agent: "MESA-Legal-Data/1.0 (+https://github.com/mesa-legal-data)" concurrency: 1 - min_interval_seconds: 15 + min_interval_seconds: 6 timeout_seconds: 30 retries: 2 max_requests_per_run: 10 max_download_bytes: 52428800 allowed_content_types: - "text/html" + - "application/pdf" + allowed_hosts: + - "bedesten.adalet.gov.tr" diff --git a/docs/BUILD_STATE.json b/docs/BUILD_STATE.json deleted file mode 100644 index f3d49b4..0000000 --- a/docs/BUILD_STATE.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "status": "mvp_closed", - "scope": "feature_freeze", - "pilot_ready": true, - "external_mesa_ready": false, - "final_audit_status": "comprehensive_repair_verified_contract_only", - "current_task": "CLOSED", - "completed": [ - "FREEZE-001", - "FREEZE-002", - "FREEZE-003", - "FREEZE-004", - "FREEZE-005", - "FREEZE-006", - "FREEZE-007", - "FREEZE-008", - "FREEZE-009", - "FREEZE-010", - "FREEZE-011", - "FREEZE-012", - "POLISH-001", - "POLISH-002", - "POLISH-003", - "POLISH-004", - "POLISH-005", - "POLISH-006", - "POLISH-007", - "POLISH-008", - "POLISH-009", - "POLISH-010", - "POLISH-011", - "POLISH-012", - "POLISH-013", - "MVP-CLOSURE-001", - "MVP-CLOSURE-002", - "MVP-CLOSURE-003", - "MVP-CLOSURE-004", - "BLOCKER-1-REAL-RESMI-GAZETE-PILOT-GENUINE-E2E", - "BLOCKER-2-OPERATOR-RETRY-RESTRICTED", - "BLOCKER-3-RUN-SCOPED-REQUEST-BUDGET", - "PROMPT-5-FINAL-ADVERSARIAL-AUDIT", - "MASTER-MVP-CLOSURE", - "POST-MASTER-INDEPENDENT-VERIFICATION", - "P1-RECORD-INSTANCE-NOT-NULL-ENFORCED", - "LEGAL-IDENTITY-AND-LOSSLESS-CHUNK-REPAIR", - "AUTHORITATIVE-TEMPORAL-VERSION-REPAIR", - "NATIVE-PUBLISH-AUTHORIZATION-BINDING", - "VERSIONED-CITATION-NATIVE-HANDOFF", - "COMPREHENSIVE-REPAIR-ROUND-2026-10-11" - ], - "last_test_command": ".venv/bin/pytest -ra --durations=20 --tb=short", - "last_test_result": "passed (413 passed, 1 cross-repo test skipped in 78.95s)", - "verification_checks": { - "ruff_format": "passed (217 files)", - "ruff_check": "passed", - "mypy": "passed (78 source files)", - "pip_audit": "passed (no known vulnerabilities)", - "pytest_not_scale": "passed (409 passed, 1 cross-repo skipped, 4 scale deselected)", - "pytest_scale": "passed (4 passed; peak RSS 171784 KiB)", - "live_mesa_contract": "LIVE_MESA_NOT_VERIFIED" - }, - "blockers": [ - "Live MESA HTTP routes, schemas, mutation semantics, server-side idempotency and SQL/vector/graph projection remain unverified because no endpoint or MESA source checkout is available.", - "Official Profile B TEST was not run and no certification result is claimed.", - "10-20 GB target capacity is not proven; only bounded synthetic scale tests were run." - ], - "polish_status": "completed", - "polish_current_task": "CLOSED", - "updated_at": "2026-10-11T12:45:00Z" -} diff --git a/docs/HARVEST_BUILD_STATE.json b/docs/HARVEST_BUILD_STATE.json deleted file mode 100644 index 75fc6b9..0000000 --- a/docs/HARVEST_BUILD_STATE.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "status": "mvp_closed", - "current_task": "CLOSED", - "pilot_ready": true, - "updated_at": "2026-08-12T23:36:00Z" - -} diff --git a/docs/MESA_DATA_FINAL_SELF_SERVICE_UX_REMEDIATION_REPORT.md b/docs/MESA_DATA_FINAL_SELF_SERVICE_UX_REMEDIATION_REPORT.md deleted file mode 100644 index 265c49d..0000000 --- a/docs/MESA_DATA_FINAL_SELF_SERVICE_UX_REMEDIATION_REPORT.md +++ /dev/null @@ -1,126 +0,0 @@ -# MESA Data — Final Self-Service UX Remediation Report - -**Date:** 2026-08-16 -**Status:** COMPLETE / ZERO-KNOWLEDGE USABILITY CERTIFIED -**Repository:** `/storage/data-law/mesa-legal-data` -**Standard:** A person who knows nothing about MESA Data must be able to open the system, collect data, inspect documents, understand problems, recover from normal validation failures, and export/use the result without terminal commands, YAML editing, raw IDs, English backend errors, or developer assistance. - ---- - -## 1. Executive Summary - -During the final narrow usability remediation pass, all remaining real user-flow defects discovered during zero-knowledge testing were completely resolved. No architectural redesign or frontend framework rewrites were introduced; existing vanilla HTML/JS and FastAPI endpoints were hardened, aligned, and tested with end-to-end regression coverage. - -All 233 automated non-scale tests (acceptance, web, integration, and unit) pass cleanly, with 100% compliance across Ruff formatting, Ruff linting, and Mypy strict static typing. - ---- - -## 2. Detailed Finding-by-Finding Remediation Audit - -### Finding 1 — Kütüphane Frontend/Backend Document Detail Contract Mismatch -* **Root Cause:** The Kütüphane view in `app.js` referenced legacy field names (`doc.status` instead of `doc.lifecycle_status`), did not render top-level `source_id`, and attempted to read document text from an absent preview property rather than fetching from the dedicated `/api/documents/{id}/text` endpoint. -* **Remediation:** - - `web/api.py`: Updated `get_document_detail` to expose top-level `source_id` alongside artifact relations. - - `web/static/app.js`: Updated `loadLibraryView` to render `statusBadge(doc.lifecycle_status || doc.status)` and `doc.source_id || "resmi_gazete"`. - - `web/static/app.js`: Updated `viewDocDetail` to concurrently query `/api/documents/{id}` and `/api/documents/{id}/text`. Real text is displayed in the primary document text container. - - Technical identifiers (Document ID, Artifact ID, SHA256) are cleanly collapsed inside `
`. -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_1_kutuphane_contract_and_text_retrieval` (PASSED). - ---- - -### Finding 2 — AYM Manual Ingestion Family & Document Identity Logic -* **Root Cause:** Manual upload and URL import forms sent `family: "legislation"` and generated legislation-scoped document IDs (`tr:legislation:...`), violating `config/sources.yaml` which designates AYM as a `decision` family source and causing `SOURCE_FAMILY_NOT_ALLOWED` errors. -* **Remediation:** - - `web/api.py`: Implemented defense-in-depth source family mapping (`SOURCE_FAMILY_MAP = {"aym": "decision", "yargitay": "decision", ...}`) in `upload_artifact` and `import_from_url`. - - `web/static/app.js`: Updated `#form-upload-file` and `#form-upload-url` to extract source family (`SOURCE_CAPABILITIES[sourceId].family`) and generate canonical case-law document IDs (`tr:case-law:{source_id}:{doc_type}:{num}`). -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_2_and_3_aym_manual_ingestion_family_and_types` (PASSED). - ---- - -### Finding 3 — Manual Ingestion Document Types Constrained by Source Capabilities -* **Root Cause:** The manual upload and URL import UI exposed static document type lists that allowed users to select invalid combinations (e.g., selecting AYM with "Kanun" or "Yönetmelik"). -* **Remediation:** - - `web/static/app.js`: Defined `SOURCE_CAPABILITIES` mapping allowed document types per source (`resmi_gazete`, `mevzuat`, `aym`). - - Implemented `updateDocTypesForSource(sourceSelectId, typeSelectId)` which dynamically populates type dropdowns upon source selection and clears incompatible previous selections with clear user toast feedback. -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_2_and_3_aym_manual_ingestion_family_and_types` (PASSED). - ---- - -### Finding 4 — Artifact-Level Issues Connected to Understandable Document Context -* **Root Cause:** Artifact-level issues (e.g., ingest errors on files before document creation or parsing failures) displayed raw SHA256 hashes as titles and lacked document navigation. -* **Remediation:** - - `web/api.py`: Updated `list_issues` SQL query to join `artifacts` and `documents` tables, resolving `document_title`, `document_id`, `source_id`, and `raw_path` for artifact subjects. - - `web/static/app.js`: Updated `loadReviewView` issues tab to display `"İşlenemeyen kaynak dosya (Kaynak Adı)"` when unlinked, render `"Belgeyi Gör"` when `document_id` exists, and hide raw IDs in collapsible technical details. -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_4_artifact_level_issue_context_and_presentation` (PASSED). - ---- - -### Finding 5 — Pipeline & Privacy Error Codes Translated into Human Turkish -* **Root Cause:** Error codes such as `TRANSPORT_VERIFICATION_FAILED`, `PARSING_FAILED`, `PRIVACY_TCKN_DETECTED`, etc., fell back to technical or English descriptions. -* **Remediation:** - - `web/static/app.js`: Expanded `humanIssueMessage(code, message)` presentation dictionary with comprehensive Turkish translations for all pipeline, privacy, schema, transport, lock, and policy codes. - - Implemented a safe human Turkish fallback (`"Belgenin işlenmesi sırasında bir sorun oluştu."`) for unmapped codes. -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_5_human_error_presentation_coverage` (PASSED). - ---- - -### Finding 6 — Harvest Zero-Document-Type Selection Blocking -* **Root Cause:** Deselecting all document types in the harvest form allowed the request to proceed and fall back to all document types unexpectedly. -* **Remediation:** - - `web/static/app.js`: Validated in `startHarvestAction` that `if (docTypes.length === 0)` displays a warning toast (`"En az bir belge türü seçmelisiniz."`) and stops submission. - - `web/api.py`: Hardened `start_harvest_endpoint` to reject empty lists (`[]`) with HTTP 400 `INVALID_DOCUMENT_TYPES`. -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_6_harvest_empty_types_rejected` (PASSED). - ---- - -### Finding 7 — Issue Resolution Semantics, Confirmation Modal & Audit Trail -* **Root Cause:** Resolving an issue was labelled as a generic action without clarifying that it represents a manual human override / acknowledgement, and lacked audit logging in the endpoint. -* **Remediation:** - - `web/static/index.html`: Added `#modal-resolve-issue` modal with clear human override explanation, reason note input, and dynamic high-severity / PII warnings. - - `web/static/app.js`: Renamed button to `"Manuel Çözüldü Kabul Et"`, opening the confirmation modal and requiring a reason note before submission. - - `web/api.py`: Updated `resolve_issue_endpoint` to log structured audit events via `log_audit_event(conn, actor=resolved_by, action="issue_resolved", subject_type="issue", ...)`. -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_finding_7_issue_manual_resolution_audit` (PASSED). - ---- - -## 3. Mandatory Cross-Flow Verification (Section 32) - -A comprehensive cross-flow test was implemented and verified: -1. **Document Ingestion & Parsing:** Document ingested and processed into catalog record. -2. **Issue Detection:** A blocking validation issue (`BLOCKING_ISSUES_EXIST`) is opened against the record. -3. **Approval Attempt Blocked:** Attempting to approve the record via `POST /api/records/{id}/approve` is blocked with HTTP 400 `BLOCKING_ISSUES_EXIST` and human Turkish explanation. -4. **Issue Inspection & Navigation:** Reviewer views issue list with resolved document/record context. -5. **Manual Override:** Reviewer executes manual resolution with justification note (`resolution_note`), creating an immutable audit log entry. -6. **Approval Succeeded:** Re-attempting approval now succeeds with status `approved`. - -* **Automated Verification:** `tests/acceptance/test_final_self_service_ux_remediation.py::test_cross_flow_blocked_review_to_issue_resolution` (PASSED). - ---- - -## 4. Test Results Summary - -| Test Suite | Commands | Status | Details | -|---|---|---|---| -| Acceptance Tests | `uv run pytest tests/acceptance/ -ra` | **PASSED** | 32 passed in 40.19s | -| Web Admin Tests | `uv run pytest tests/web/ -ra` | **PASSED** | 1 passed in 0.45s | -| Integration Tests | `uv run pytest tests/integration/ -ra` | **PASSED** | 36 passed in 21.32s | -| Unit Tests | `uv run pytest tests/unit/ -ra` | **PASSED** | 165 passed in 6.23s | -| Complete Non-Scale Suite | `uv run pytest -k "not scale" -ra` | **PASSED** | 233 passed in 14.33s | -| Code Formatting | `uv run ruff format --check .` | **PASSED** | 178 files formatted | -| Code Linting | `uv run ruff check .` | **PASSED** | 0 errors | -| Static Type Checking | `uv run mypy src` | **PASSED** | 0 errors in 67 files | -| JS Static Syntax | `node -c src/mesa_legal_data/web/static/*.js` | **PASSED** | 0 syntax errors | - ---- - -## 5. Architectural Integrity & Scope Verification - -- **Hard Ban Adherence:** No React, Vue, Svelte, Next.js, Tailwind migration, Redis, Celery, Kafka, RQ, WebSockets, SSE, new routing frameworks, new state engines, or OCR projects were introduced. -- **Data Model Integrity:** Preserved canonical JSONL, SQLite catalog schema, and atomic locking. -- **Zero-Knowledge Usability:** All user-facing views operate exclusively in clear Turkish terminology, with technical IDs sequestered in secondary disclosure panels and comprehensive inline recovery guidance. - ---- - -## 6. Final Certification - -**MESA Data Self-Service UX Remediation is complete, certified, and ready for zero-knowledge end-user operation.** diff --git a/docs/MESA_DATA_PRODUCT_UX_FINAL_CERTIFICATION_AUDIT.md b/docs/MESA_DATA_PRODUCT_UX_FINAL_CERTIFICATION_AUDIT.md deleted file mode 100644 index 7a0a547..0000000 --- a/docs/MESA_DATA_PRODUCT_UX_FINAL_CERTIFICATION_AUDIT.md +++ /dev/null @@ -1,363 +0,0 @@ -# MESA Data Product / UX Closure — Final Certification Audit - -**Audit Date:** 2026-08-16 -**Auditor:** Antigravity Independent Certification Subagent -**Authoritative Specification:** `MESA_DATA_PRODUCT_UX_CLOSURE_AGENT_REPORT.md` -**Repository Path:** `/storage/data-law/mesa-legal-data` - ---- - -## 1. Executive Verdict - -### **CONDITIONAL GO (Ready for Final Release after 2 Endpoint Route Aliases)** - -The MESA Data Product / UX Closure implementation has successfully transformed the platform from an internal developer console into an accessible, human-first legal data product. All core foundations are in place: -1. **One-command web runtime startup** (`uv run mesa-data web` with idempotent bootstrapping and automated recovery). -2. **5-item Information Architecture** (`Ana Sayfa`, `Veri Topla`, `Kütüphane`, `İnceleme`, `Dışa Aktar` + collapsible `Gelişmiş`). -3. **Automated Resmî Gazete collection** with date-coverage progress, safety bounds, and cancellation. -4. **Clean-room zero-knowledge onboarding** and human terminology translation across all primary screens. -5. **Preserved security, TLS allowlisting, and verification state machines**. - -Two specific client-to-server route naming discrepancies were identified during deep contract tracing (`MUX-CERT-001` on MESA transfer and `MUX-CERT-002` on manual URL import) along with one type attribute mismatch (`MUX-CERT-003`), each requiring a 1-line alias fix. - ---- - -## 2. Repository Baseline - -- **Repository Path:** `/storage/data-law/mesa-legal-data` -- **Branch:** `main` -- **HEAD Commit:** `7ac917ffddccdb0f0c7a80abc4735b1d3efa8d46` -- **Worktree State:** Clean functional diff containing all Product / UX Closure implementation files: - - Modified: `docs/HIZLI_BASLANGIC.md`, `docs/KULLANIM_KILAVUZU.md`, `src/mesa_legal_data/catalog.py`, `src/mesa_legal_data/cli.py`, `src/mesa_legal_data/harvest/cli.py`, `src/mesa_legal_data/operations.py`, `src/mesa_legal_data/web/api.py`, `src/mesa_legal_data/web/schemas.py`, `src/mesa_legal_data/web/static/app.js`, `src/mesa_legal_data/web/static/index.html`, `src/mesa_legal_data/web/static/styles.css`, `src/mesa_legal_data/web/static/ui.js` - - Created: `docs/UX_CLOSURE_STATE.json`, `src/mesa_legal_data/harvest/service.py`, `src/mesa_legal_data/web/bootstrap.py`, `tests/acceptance/test_web_user_journey_contract.py`, `tests/unit/harvest/test_service_orchestration.py`, `tests/unit/test_web_bootstrap.py` - ---- - -## 3. Specification Reviewed - -The complete authoritative report `MESA_DATA_PRODUCT_UX_CLOSURE_AGENT_REPORT.md` (1905 lines, sections 0 through 27) was reviewed in full and used as the benchmark for this audit. - ---- - -## 4. Build / State Findings - -- **Mechanism Used:** Reused existing `operation_jobs` in `catalog.sqlite` and `discovery_cursors` / `harvest_items` in `harvest.sqlite`. -- **New Infrastructure:** None. No new database tables or worker subsystems were added. -- **Interruption / Recovery:** `prepare_web_runtime()` invokes `recover_interrupted_operations(conn)` before launching web server, resetting dangling `running` jobs to `interrupted`. -- **Implementation Tracking:** Tracked cleanly via `docs/UX_CLOSURE_STATE.json`. - ---- - -## 5. Over-Engineering Assessment - -- **Unnecessary Infrastructure Introduced:** **NO** -- **Evidence:** - - Frontend frameworks (React/Vue/Next/Tailwind): **None** (Vanilla HTML/JS/CSS preserved). - - External task queues (Celery/Redis/Kafka): **None** (`operations.py` single worker preserved). - - Background schedulers (cron/APScheduler): **None** (Explicit user actions used). - - Real-time transport (WebSockets/SSE): **None** (4-second polling only during active harvest). - - AI/LLM/OCR pipelines: **None**. - ---- - -## 6. Phase Certification - -| Phase | Description | Status | Evidence | -|---|---|---|---| -| **Phase 0** | Repair existing frontend/API contracts | **PASS** | `catalog.py` export list, enriched review endpoints, truth table on `/sources`. | -| **Phase 1** | One-command web startup (`prepare_web_runtime`) | **PASS** | `tests/unit/test_web_bootstrap.py` (3 passed), clean data root test passed. | -| **Phase 2** | Extract harvest service (`harvest/service.py`) | **PASS** | `tests/unit/harvest/test_service_orchestration.py` (4 passed), CLI delegation clean. | -| **Phase 3** | Harvest collection as operation job | **PASS** | `operations.py` supports `harvest_collection`, cancellation safety verified. | -| **Phase 4** | Minimal Harvest Web API (`/harvest/status`, `/start`, `/stop`) | **PASS** | Duplicate 409 guard, date validation, and progress calculations verified. | -| **Phase 5** | Rebuild Information Architecture & Views | **PASS** | 5-item primary nav + Gelişmiş, Onboarding welcome, human dictionaries. | -| **Phase 6** | Acceptance journeys and contract tests | **PASS** | `tests/acceptance/test_web_user_journey_contract.py` (6 passed). | -| **Phase 7** | Documentation & User Guides | **PASS** | `HIZLI_BASLANGIC.md` & `KULLANIM_KILAVUZU.md` updated with human-first workflows. | -| **Phase 8** | Regression & clean verification | **PASS** | Full test suite: **219 passed clean** (0 failures). | - ---- - -## 7. Frontend/API Contract Results - -1. **Documents (`GET /api/documents`)**: - - Uses `page` and `page_size`. - - Returns `items`, `total`, `page`, `page_size`. - - Enriched with `source_id` subquery for frontend display and filtering. - - **Result: PASS** - -2. **Reviews (`GET /api/records` & `GET /api/records/{id}`)**: - - Consumes `data.items` array correctly. - - Response enriched with `document_title`, `text_preview`, and `validation_status`. - - `approve` and `reject` actions work with optional reviewer note. - - **Result: PASS** - -3. **Sources (`GET /api/sources`)**: - - Separated from `/api/config/public`. - - Returns truthful capability array (`resmi_gazete: supported`, `mevzuat: manual`, `aym: manual`, `yargitay: disabled`). - - **Result: PASS** - -4. **Exports (`GET /api/exports`)**: - - Implemented `list_export_packages` in `catalog.py`. - - `GET /api/exports` returns newest-first export packages. - - Downloads work via `/api/exports/{id}/download`. - - **Result: PASS** - -5. **Audit (`GET /api/audit-events`)**: - - Frontend correctly calls `/api/audit-events?limit=50`. - - **Result: PASS** - -6. **Explorer (`GET /api/explorer/search`)**: - - Uses `effective_page_size` in SQL `LIMIT` clause. - - Whitelist-based sorting supported on `created_at`, `record_id`, `record_type`. - - Moved to Gelişmiş. - - **Result: PASS** - -7. **Release State Machine**: - - Enforces `created -> verified -> published -> import`. - - Cannot publish unverified release. - - **Result: PASS** - ---- - -## 8. Clean Startup Result - -- **Test:** Started web runtime against an isolated, non-existent temporary directory without prior execution of `mesa-data init` or `mesa-data migrate`. -- **Verification:** - - `data_root` directory structure created (`raw`, `canonical`, `releases`, `tmp`). - - `catalog.sqlite` created and 18 catalog tables migrated. - - `harvest/harvest.sqlite` created and 7 harvest tables migrated. - - Interrupted operations recovered. - - Second run completed identically with 100% idempotency. -- **Result: PASS** - ---- - -## 9. Harvest Result - -- **Shared Logic:** Pure Python service functions `run_discovery_once` and `run_collection_until_pause` in `src/mesa_legal_data/harvest/service.py`. -- **Start / Stop:** Minimal web API endpoints `POST /api/harvest/start` and `POST /api/harvest/stop` with duplicate 409 conflict guard. -- **Safety Bounds:** Exits cleanly on low disk space (`LOW_DISK_SPACE`), daily budgets, cursor caught up (`NO_QUEUED_ITEMS`), or user cancellation without infinite sleep loops. -- **Result: PASS** - ---- - -## 10. UX / Information Architecture - -- **Primary Navigation (5 items):** - 1. `Ana Sayfa` (Overview, Onboarding, Recommended Next Action, Top Stats, Harvest Progress) - 2. `Veri Topla` (Resmî Gazete card, manual file/URL ingestion, honest source capabilities) - 3. `Kütüphane` (Document browsing, search & filters, text preview, technical details disclosure) - 4. `İnceleme` (Pending records table, issues table, text preview review modal) - 5. `Dışa Aktar` (JSONL / CSV export creation, export history table, MESA transfer workflow) -- **Secondary Navigation:** - - `Gelişmiş` collapsible disclosure containing `Kaynaklar`, `Veri Gezgini`, `Release Geçmişi`, `Arka Plan İşlemleri`, `İşlem Geçmişi`, and `Sistem`. -- **Result: PASS** - ---- - -## 11. First-Run Experience - -- **Zero-Data State:** - - When `documents == 0` and harvest is `not_started`, the application displays the welcoming `MESA Data’ya hoş geldiniz` banner. - - Explains the 3-step mental model: *1. Veri topla*, *2. İncele*, *3. Dışa aktar*. - - Provides two clear starting actions: `İlk veri toplamayı başlat` and `Dosya yükle`. - - Hides empty database counter tables and technical internal IDs. -- **Result: PASS** - ---- - -## 12. Technical-Language Leakage - -- Technical concepts (*Artifact, Canonical, Pipeline, Cursor, Backfill, Lease*) have been translated or moved behind collapsed `
` summaries across all primary screens. -- Central presentation helpers `humanTerm()` and `statusBadge()` in `app.js` translate backend enum values into clear Turkish terminology. -- **Result: PASS** - ---- - -## 13. Manual Upload - -- Unified file upload form accepts source, document type, title, document number, and file. -- Client automatically calls `POST /api/artifacts/{id}/process` immediately after upload, eliminating the need to find an Artifact ID or navigate to a separate screen. -- **Result: PASS** - ---- - -## 14. Kütüphane - -- Default columns: *Başlık*, *Tür*, *Kaynak*, *Durum*, *Güncelleme*, *İşlem*. -- Text preview modal renders document contents. -- Document and artifact identifiers are neatly tucked under collapsible *Teknik ayrıntılar*. -- **Result: PASS** - ---- - -## 15. İnceleme - -- Split tabs for *İnceleme bekleyenler* and *Sorunlar*. -- Review modal shows full text preview and validation status. -- Single-click *Onayla* and *Reddet* actions do not require typing a reviewer name. -- **Result: PASS** - ---- - -## 16. Export - -- Clean selection for JSONL (*Veri işleme ve model eğitiminde önerilen*) and CSV (*Excel ve tablo araçları için*). -- Export history table displays package ID, format, status (*Hazır*), record count, and download link. -- **Result: PASS** - ---- - -## 17. MESA Transfer - -- Automated 4-step sequence: `build -> verify -> publish -> import`. -- Step-by-step progress labels displayed (*1/4 Paket hazırlanıyor...*, *2/4 Bütünlük doğrulanıyor...*, *3/4 Yayına alınıyor...*, *4/4 MESA aktarım alanına aktarılıyor...*). -- **Notice:** Endpoint route alias needed for step 4 (see `MUX-CERT-001`). -- **Result: CONDITIONAL PASS (Pending route alias)** - ---- - -## 18. Error UX - -- Human-friendly Turkish error messages provide immediate actionable guidance for low disk space, already-running collectors, and empty export sets. -- **Result: PASS** - ---- - -## 19. Dead Controls - -- All primary buttons, filters, pagination controls, and tabs are wired to active event listeners and API handlers. -- **Result: None found in audited flows.** - ---- - -## 20. Accessibility / Responsive - -- WCAG-compatible semantics preserved (`
`, skip-link, ARIA modal dialogs, `:focus-visible`, `prefers-reduced-motion`). -- Mobile sidebar drawer and responsive overflow tables verified on narrow viewports. -- **Result: PASS** - ---- - -## 21. Security Regression - -- Geotrust intermediate CA, TLS validation, source allowlisting, localhost security checks, and token authentication on non-loopback binds preserved intact. -- **Result: PASS** - ---- - -## 22. Test Results - -| Test Suite | Command | Duration | Result | Status | -|---|---|---|---|---| -| **Web Bootstrap** | `uv run pytest tests/unit/test_web_bootstrap.py -ra` | 0.29s | 3 passed | **PASS** | -| **Harvest Service** | `uv run pytest tests/unit/harvest/test_service_orchestration.py -ra` | 0.52s | 4 passed | **PASS** | -| **Visual Contract** | `uv run pytest tests/acceptance/test_web_visual_contract.py -ra` | 0.34s | 3 passed | **PASS** | -| **Web Journey Contract** | `uv run pytest tests/acceptance/test_web_user_journey_contract.py -ra` | 0.87s | 6 passed | **PASS** | -| **Acceptance Suite** | `uv run pytest tests/acceptance/ -ra` | 40.00s | 18 passed | **PASS** | -| **Web Suite** | `uv run pytest tests/web/ -ra` | 0.48s | 1 passed | **PASS** | -| **Security Suite** | `uv run pytest tests/security/ -ra` | 0.49s | 3 passed | **PASS** | -| **Integration Suite** | `uv run pytest tests/integration/ -ra` | 20.26s | 36 passed | **PASS** | -| **Unit Suite** | `uv run pytest tests/unit/ -ra` | 6.47s | 165 passed | **PASS** | -| **Full Regression** | `uv run pytest -k "not scale" -ra` | 12.39s | 219 passed | **PASS** | - ---- - -## 23. Acceptance Journey - -| Step | User Action / Expectation | Result | Evidence | -|---|---|---|---| -| **1** | User opens MESA Data | **PASS** | `GET /api/health` returns 200 OK. | -| **2** | User understands what application does | **PASS** | First-run welcome banner displays product description. | -| **3** | User understands 3 basic actions (Veri Topla, İncele, Dışa Aktar) | **PASS** | 3-step action cards rendered on Home screen. | -| **4** | User clicks first-data-collection action | **PASS** | `İlk veri toplamayı başlat` navigates to Veri Topla. | -| **5** | User understands Resmî Gazete collection scope | **PASS** | Scope displayed as *1 Ocak 2015’ten bugüne* with 5 Turkish doc types. | -| **6** | User starts collection without terminal/YAML/internal IDs | **PASS** | `POST /api/harvest/start` launches collection. | -| **7** | User sees understandable collection status/progress | **PASS** | Date coverage progress bar and current cursor date rendered. | -| **8** | User can stop collection safely | **PASS** | `POST /api/harvest/stop` halts collection cleanly. | -| **9** | User can browse collected documents from Kütüphane | **PASS** | `GET /api/documents` renders document list with title, type, source. | -| **10** | User can open document without manipulating internal ID | **PASS** | `Detay` button opens modal with text preview. | -| **11** | User can identify records requiring review | **PASS** | `İnceleme` view renders pending records with document titles. | -| **12** | User can understand review content | **PASS** | Review modal displays readable text preview. | -| **13** | User can approve/reject | **PASS** | `Onayla` and `Reddet` buttons work with 1 click. | -| **14** | User can create/download an export | **PASS** | JSONL/CSV packages created and downloaded from history table. | -| **15** | User can start MESA transfer without understanding release state | **PASS** | Client orchestrates 4-step sequence (see `MUX-CERT-001` alias). | - ---- - -## 24. Requirement Traceability Matrix - -| Requirement | Spec Section | Implementation Evidence | Test Evidence | Status | -|---|---|---|---|---| -| One-command web bootstrap | Section 4 | `src/mesa_legal_data/web/bootstrap.py` | `test_web_bootstrap.py` | **PASS** | -| Extract harvest service | Section 5 | `src/mesa_legal_data/harvest/service.py` | `test_service_orchestration.py` | **PASS** | -| Background collection operation | Section 6 | `src/mesa_legal_data/operations.py` | `test_service_orchestration.py` | **PASS** | -| Harvest Web API | Section 7 | `src/mesa_legal_data/web/api.py` | `test_web_user_journey_contract.py` | **PASS** | -| 5-item Information Architecture | Section 8 | `src/mesa_legal_data/web/static/index.html` | `test_web_visual_contract.py` | **PASS** | -| First-run onboarding banner | Section 9 | `src/mesa_legal_data/web/static/index.html` | `test_web_visual_contract.py` | **PASS** | -| Veri Topla Resmî Gazete card | Section 10 | `src/mesa_legal_data/web/static/index.html` | `test_web_visual_contract.py` | **PASS** | -| Automatic upload-and-process | Section 11 | `src/mesa_legal_data/web/static/app.js` | Integration test verified | **PASS** | -| Simplified Kütüphane list | Section 12 | `src/mesa_legal_data/web/static/app.js` | `test_web_user_journey_contract.py` | **PASS** | -| Simplified İnceleme review | Section 13 | `src/mesa_legal_data/web/static/app.js` | `test_web_user_journey_contract.py` | **PASS** | -| Simplified Dışa Aktar & history | Section 14 | `src/mesa_legal_data/web/static/app.js` | `test_web_user_journey_contract.py` | **PASS** | -| Central human terminology map | Section 15 | `src/mesa_legal_data/web/static/app.js` | Visual inspections | **PASS** | -| Friendly error guidance | Section 16 | `src/mesa_legal_data/web/static/app.js` | Visual inspections | **PASS** | -| Truthful source capabilities | Section 18 | `src/mesa_legal_data/web/api.py` | `test_web_user_journey_contract.py` | **PASS** | - ---- - -## 25. Defects - -### P0 — Certification Blockers -*None.* - -### P1 — Core Workflow Defect Findings - -#### `MUX-CERT-001` -- **Severity:** P1 -- **Requirement:** Section 14.2 / Section 22 ("MESA'ya Aktar" client-side sequence orchestration). -- **Relevant Files:** `src/mesa_legal_data/web/static/app.js` (lines 605, 718) vs `src/mesa_legal_data/web/api.py` (lines 1310-1313). -- **Description:** In `app.js`, step 4 of `runMesaTransferSequence` calls `POST /api/releases/${releaseId}/import-staging`. The backend router in `api.py` registers `/releases/{release_id:path}/import-to-staging` and `/releases/{release_id:path}/import` (without the standalone `/import-staging` route), causing HTTP 404 when invoked from the client sequence. -- **Expected:** `POST /api/releases/${releaseId}/import-staging` succeeds with 200 OK. -- **Recommended Remediation:** In `src/mesa_legal_data/web/api.py`, add `@router.post("/releases/{release_id:path}/import-staging")` decorator above `import_release_endpoint`. - -#### `MUX-CERT-002` -- **Severity:** P1 -- **Requirement:** Section 11 (Manual URL import form). -- **Relevant Files:** `src/mesa_legal_data/web/static/app.js` (line 865) vs `src/mesa_legal_data/web/api.py` (line 981). -- **Description:** Submitting the manual URL import form in `app.js` calls `POST /api/documents/import-url`. The backend route is registered under `POST /api/manual/import-url`, resulting in HTTP 405. -- **Expected:** `POST /api/documents/import-url` succeeds with 200 OK. -- **Recommended Remediation:** In `src/mesa_legal_data/web/api.py`, add `@router.post("/documents/import-url")` decorator above `import_from_url`. - -### P2 — Configuration / Type Safety Notice - -#### `MUX-CERT-003` -- **Severity:** P2 -- **Requirement:** Section 7.2 (Harvest start runtime overrides). -- **Relevant Files:** `src/mesa_legal_data/operations.py` (line 161). -- **Description:** In `_run_operation_task`, runtime override assigns `src_cfg.document_types = doc_types`. In `HarvestSourceConfig`, the target field is `src_cfg.selection.allowed_document_types`. -- **Recommended Remediation:** Change `src_cfg.document_types = doc_types` to `src_cfg.selection.allowed_document_types = doc_types`. - -### P3 — Polish & Style Notices - -#### `MUX-CERT-004` -- **Severity:** P3 -- **Description:** 7 minor import formatting/unused import notices reported by `ruff check` in test files and `bootstrap.py`. -- **Recommended Remediation:** Run `uv run ruff check --fix`. - ---- - -## 26. Final Zero-Knowledge User Verdict - -> **Can a first-time user who knows nothing about MESA Data successfully perform the core workflow without documentation, terminal commands, YAML editing or knowledge of internal IDs?** - -**YES.** - -A completely new user is greeted with a clear 3-step value proposition and can immediately launch automated Resmî Gazete data collection with a single click. The user sees understandable date-coverage progress, reviews readable document previews in Turkish, and creates standard exports without ever encountering internal IDs or database schemas. All underlying safety guards, verification sequences, and storage limits operate seamlessly underneath. - ---- - -## 27. Final Certification - -**Verdict:** **CONDITIONAL GO** - -**Reason:** All 9 closure phases are implemented, fully tested, and verified with 219 passing unit/integration/acceptance tests. The system is certified ready for final release upon applying the two single-line endpoint route aliases documented in `MUX-CERT-001` and `MUX-CERT-002`. diff --git a/docs/MESA_DATA_PRODUCT_UX_FINAL_REMEDIATION_REPORT.md b/docs/MESA_DATA_PRODUCT_UX_FINAL_REMEDIATION_REPORT.md deleted file mode 100644 index fe89e59..0000000 --- a/docs/MESA_DATA_PRODUCT_UX_FINAL_REMEDIATION_REPORT.md +++ /dev/null @@ -1,153 +0,0 @@ -# MESA Data Product / UX Closure — Final Remediation Report - -**Date:** 2026-08-16 -**Auditor / Remediation Agent:** Antigravity Remediation Subagent -**Authoritative Audit Report:** `MESA_DATA_PRODUCT_UX_FINAL_CERTIFICATION_AUDIT.md` -**Repository:** `/storage/data-law/mesa-legal-data` - ---- - -## 1. Final Status - -**`COMPLETE`** - -All four audit findings (`MUX-CERT-001`, `MUX-CERT-002`, `MUX-CERT-003`, `MUX-CERT-004`) have been remediated with minimal, safe code modifications. Explicit regression test coverage and an API contract guard have been added. All 223 tests pass with 100% success rate, static analysis (ruff & mypy) is clean, and the three affected user journeys have been verified end-to-end. - ---- - -## 2. Repository State - -- **Branch:** `main` -- **Starting HEAD:** `7ac917ffddccdb0f0c7a80abc4735b1d3efa8d46` -- **Ending HEAD:** `7ac917ffddccdb0f0c7a80abc4735b1d3efa8d46` (working tree cleanly modified in-place) -- **Worktree Status:** - ```text - M docs/HIZLI_BASLANGIC.md - M docs/KULLANIM_KILAVUZU.md - M src/mesa_legal_data/catalog.py - M src/mesa_legal_data/cli.py - M src/mesa_legal_data/harvest/cli.py - M src/mesa_legal_data/operations.py - M src/mesa_legal_data/web/api.py - M src/mesa_legal_data/web/schemas.py - M src/mesa_legal_data/web/static/app.js - M src/mesa_legal_data/web/static/index.html - M src/mesa_legal_data/web/static/styles.css - M src/mesa_legal_data/web/static/ui.js - ?? docs/UX_CLOSURE_STATE.json - ?? src/mesa_legal_data/harvest/service.py - ?? src/mesa_legal_data/web/bootstrap.py - ?? tests/acceptance/test_web_user_journey_contract.py - ?? tests/unit/harvest/test_service_orchestration.py - ?? tests/unit/test_web_bootstrap.py - ``` - ---- - -## 3. Finding Resolution - -| Finding | Before | Fix | Evidence | Status | -|---|---|---|---|---| -| **`MUX-CERT-001`** | Frontend MESA transfer called `/api/releases/{id}/import-staging`, which returned HTTP 404 because backend only had `/import-to-staging` and `/import`. | Added `@router.post("/releases/{release_id:path}/import-staging")` route alias in `src/mesa_legal_data/web/api.py` pointing to `import_release_endpoint`. | `test_regression_mux_cert_001_mesa_transfer_route` passes; end-to-end 4-step sequence returns 200 `imported`. | **FIXED** | -| **`MUX-CERT-002`** | Frontend manual URL import called `/api/documents/import-url`, which returned HTTP 405 Method Not Allowed because backend route was `/api/manual/import-url`. | Added `@router.post("/documents/import-url")` route alias in `src/mesa_legal_data/web/api.py` pointing to `import_from_url`. | `test_regression_mux_cert_002_manual_url_import_route` passes; invalid hosts safely rejected with 400 `URL_IMPORT_FAILED`. | **FIXED** | -| **`MUX-CERT-003`** | `operations.py` assigned `src_cfg.document_types = doc_types`, triggering mypy error and bypassing `selection.allowed_document_types`. | Updated assignment to `src_cfg.selection.allowed_document_types = doc_types` in `src/mesa_legal_data/operations.py`. | `test_regression_mux_cert_003_harvest_document_type_filtering` passes; `mypy src` passes with 0 errors across 67 files. | **FIXED** | -| **`MUX-CERT-004`** | `ruff check` reported 7 import sorting (I001) and unused import (F401) warnings in newly added test files and `bootstrap.py`. | Ran `ruff check --fix` on affected files; cleaned up unused imports and formatted import blocks. | `uv run ruff check src tests` returns *All checks passed!* | **FIXED** | - ---- - -## 4. Files Changed - -- **`src/mesa_legal_data/web/api.py`**: Added route decorators `@router.post("/documents/import-url")` and `@router.post("/releases/{release_id:path}/import-staging")` to wire frontend client actions to existing verified handlers without duplicating business logic. -- **`src/mesa_legal_data/operations.py`**: Updated `src_cfg.selection.allowed_document_types = doc_types` to properly enforce document type filtering in the harvest collection worker. -- **`src/mesa_legal_data/web/bootstrap.py`**: Formatted import block to satisfy `ruff` import ordering rules. -- **`tests/acceptance/test_web_user_journey_contract.py`**: Added 3 explicit regression tests (`test_regression_mux_cert_001_mesa_transfer_route`, `test_regression_mux_cert_002_manual_url_import_route`, `test_regression_mux_cert_003_harvest_document_type_filtering`) and 1 contract guard test (`test_frontend_api_contract_guard`). -- **`tests/unit/harvest/test_service_orchestration.py`**: Cleaned up unused imports and formatted import blocks. -- **`tests/unit/test_web_bootstrap.py`**: Cleaned up unused imports and formatted import blocks. -- **`docs/UX_CLOSURE_STATE.json`**: Updated implementation tracker with remediated findings and final test verification counts. - ---- - -## 5. Regression Tests Added - -1. **`test_regression_mux_cert_001_mesa_transfer_route`**: - - **Defect Protected:** `MUX-CERT-001` (MESA Transfer 404 on final step). - - **Why Old Suite Missed It:** Previous tests directly called internal backend paths (`/import` or `/import-to-staging`) rather than the frontend client route (`/import-staging`). - - **Coverage:** Verifies route exists, rejects unbuilt release (404), rejects unverified/unpublished release (409), and imports published release (200 OK). - -2. **`test_regression_mux_cert_002_manual_url_import_route`**: - - **Defect Protected:** `MUX-CERT-002` (Manual URL Import 405). - - **Why Old Suite Missed It:** Previous integration tests exercised `/manual/import-url` or direct service functions rather than the frontend client route `/documents/import-url`. - - **Coverage:** Verifies route exists (no 404/405) and enforces URL security policy against unauthorized hosts. - -3. **`test_regression_mux_cert_003_harvest_document_type_filtering`**: - - **Defect Protected:** `MUX-CERT-003` (Harvest document types ignored in runtime config). - - **Why Old Suite Missed It:** Unit tests tested default config without dynamic request overrides through `operations.py`. - - **Coverage:** Verifies that passing `["law", "regulation"]` populates `src_cfg.selection.allowed_document_types` and excludes non-selected types. - -4. **`test_frontend_api_contract_guard`**: - - **Defect Protected:** Any future frontend/backend route mismatch regression. - - **Coverage:** Statically asserts that all 17 critical endpoints and methods invoked by `app.js` are registered in the FastAPI router. - ---- - -## 6. User Flow Verification - -### Flow A — Manual URL Import: **`PASS`** -- **Evidence:** `POST /api/documents/import-url` with untrusted host returns 400 `URL_IMPORT_FAILED`. Route is actively bound to `import_manual_url` and processes downloads through existing pipeline. - -### Flow B — Document Type Selection: **`PASS`** -- **Evidence:** `POST /api/harvest/start` with `document_types=["law", "regulation"]` submits background operation and configures `src_cfg.selection.allowed_document_types` dynamically in `operations.py`. - -### Flow C — MESA Transfer: **`PASS`** -- **Evidence:** Full 4-step sequence (`build` -> `verify` -> `publish` -> `import-staging`) executed against live TestClient in isolated clean data root; returns 200 OK `{"status": "imported"}` on `POST /api/releases/{id}/import-staging`. - ---- - -## 7. Test Results - -| Suite / Command | Result | Pass | Fail | Deselected | Duration | -|---|---|---|---|---|---| -| `uv run ruff check src tests` | **PASS** | All clean | 0 | 0 | 0.05s | -| `uv run mypy src` | **PASS** | 0 issues (67 files) | 0 | 0 | 0.65s | -| `uv run pytest tests/unit/test_web_bootstrap.py -ra` | **PASS** | 3 | 0 | 0 | 0.29s | -| `uv run pytest tests/unit/harvest/test_service_orchestration.py -ra` | **PASS** | 4 | 0 | 0 | 0.52s | -| `uv run pytest tests/acceptance/test_web_user_journey_contract.py -ra` | **PASS** | 10 | 0 | 0 | 3.20s | -| `uv run pytest tests/acceptance/ -ra` | **PASS** | 22 | 0 | 0 | 39.61s | -| `uv run pytest tests/web/ -ra` | **PASS** | 1 | 0 | 0 | 0.40s | -| `uv run pytest tests/security/ -ra` | **PASS** | 3 | 0 | 0 | 0.45s | -| `uv run pytest tests/integration/ -ra` | **PASS** | 36 | 0 | 0 | 20.22s | -| `uv run pytest tests/unit/ -ra` | **PASS** | 165 | 0 | 0 | 5.92s | -| **Full Suite:** `uv run pytest -k "not scale" -ra` | **PASS** | **223** | **0** | **4** | **17.15s** | - ---- - -## 8. Security Regression - -- **Release State Machine:** Strictly preserved. Unverified or unpublished releases cannot be imported via `/import-staging` (returns 409 `RELEASE_NOT_PUBLISHED`). -- **URL & Source Security:** Strictly preserved. Host allowlists and content-type restrictions remain enforced on `/documents/import-url`. -- **TLS & Certificate Validation:** GeoTrust intermediate CA and host validation intact. -- **Harvest Safety Bounds:** Disk space guards, rate limits, and cooperative cancellation preserved. - ---- - -## 9. Over-Engineering Check - -> **Was any new framework, service, worker system, scheduler, database subsystem or generic abstraction introduced?** - -**`NO`** - -The remediation was achieved with minimal, surgical changes: 2 route aliases in `api.py`, 1 attribute fix in `operations.py`, import cleanup, and targeted regression tests. - ---- - -## 10. Remaining Findings - -**`None.`** - -All four findings from the certification audit are completely resolved. - ---- - -## 11. Final Release Recommendation - -**`READY FOR FINAL CERTIFICATION`** diff --git a/docs/MESA_DATA_ZERO_KNOWLEDGE_USABILITY_CLOSURE_REPORT.md b/docs/MESA_DATA_ZERO_KNOWLEDGE_USABILITY_CLOSURE_REPORT.md deleted file mode 100644 index 9998c80..0000000 --- a/docs/MESA_DATA_ZERO_KNOWLEDGE_USABILITY_CLOSURE_REPORT.md +++ /dev/null @@ -1,127 +0,0 @@ -# MESA Data — Zero-Knowledge Usability Closure Report - -## 1. Final Status -**COMPLETE** - ---- - -## 2. Repository State -- **Branch:** `main` -- **Starting HEAD:** `00e202ad339a15024761f93ad8b924250254dec9` -- **Ending HEAD:** Working tree updated, verified, and ready for commit. -- **Worktree:** Clean after formatting and testing. - ---- - -## 3. Finding Resolution - -| # | Finding | Fix Summary | Evidence | Status | -|---|---|---|---|---| -| 1 | **Broken `/api/issues` Contract** | Handled both direct array & object payloads, enriched SQL query with `document_title` subqueries. | `test_issues_contract_and_resolution` in `tests/acceptance/test_web_user_journey_contract.py` | **FIXED** | -| 2 | **Missing Issue → Context & Resolution Workflow** | Added actionable buttons ("Kaydı İncele", "Belgeyi Gör", "Çözüldü İşaretle"), backend `POST /api/issues/{id}/resolve` endpoint, and direct link from blocked records. | Tested in `test_issues_contract_and_resolution` and manual UI inspection. | **FIXED** | -| 3 | **Technical Blocker/Review Error Messages** | Mapped `BlockingValidationIssueExists` exception to code `BLOCKING_ISSUES_EXIST` with human-first Turkish message: *"Bu kayıt henüz onaylanamaz. Çözülmesi gereken doğrulama sorunları bulunuyor."* and a direct *"Sorunları Gör"* button. | `test_blocked_record_approval_ux_contract` in `tests/acceptance/test_web_user_journey_contract.py` | **FIXED** | -| 4 | **Unsafe Defaults in Manual Upload & URL Ingestion** | Replaced preselected defaults with disabled placeholder options (`— Kaynak seçin —`, `— Belge türünü seçin —`) and added frontend form validation preventing submission without explicit selection. | Checked in `index.html` & `app.js` submit handlers. | **FIXED** | -| 5 | **`Yönetici Erişimi` Visible in Normal Header** | Hidden top header admin token button for zero-knowledge local users; moved token configuration card under `Gelişmiş → Sistem` (`#btn-sys-token`). | Verified in `index.html` and `app.js`. | **FIXED** | -| 6 | **Weak Empty States with No Next-Action CTA** | Replaced empty text with structured empty state cards featuring clear explanations and direct action buttons for Kütüphane, İnceleme (Pending), Sorunlar (Issues), and Dışa Aktar (Export). | Verified in `app.js` and `index.html`. | **FIXED** | -| 7 | **Broken `Arka Plan İşlemleri` Contract** | Updated frontend `loadOperationsView` to call `GET /api/operations/jobs` and read `finished_at` (with fallback to `completed_at`). | `test_operations_jobs_contract` and contract guard in `test_web_user_journey_contract.py`. | **FIXED** | - ---- - -## 4. Issue Workflow -```text -Issue Detected -→ User sees human-readable issue in "İnceleme → Sorunlar" (or via "Sorunları Gör" from a blocked review) -→ User understands what happened in plain Turkish (e.g., "Belgedeki tarih bilgisi okunamadı veya eksik.") -→ User clicks "Kaydı İncele" or "Belgeyi Gör" to immediately navigate to the relevant context -→ User performs available recovery (e.g. edits/re-uploads document or marks issue resolved via "Çözüldü İşaretle") -``` -All standard validation issues have direct context navigation and resolution capabilities. - ---- - -## 5. User-Facing Error Mapping -The following technical error codes and exceptions are translated into plain Turkish: -- `BLOCKING_ISSUES_EXIST` / `BlockingValidationIssueExists` → *"Bu kayıt henüz onaylanamaz. Çözülmesi gereken doğrulama sorunları bulunuyor."* -- `VALIDATION_DATE_MISSING` → *"Belgedeki tarih bilgisi okunamadı veya eksik."* -- `VALIDATION_TITLE_MISSING` → *"Belge başlığı tespit edilemedi."* -- `VALIDATION_SCHEMA_INVALID` → *"Belge yapısı veya şema doğrulaması başarısız."* -- `HASH_MISMATCH` → *"Veri bütünlüğü doğrulaması uyuşmadı (SHA256)."* -- `CANONICAL_LINE_MISSING` → *"Kanonik veri dosyasında ilgili kayıt satırı bulunamadı."* -- `DUPLICATE_ITEM` → *"Aynı içerikli mükerrer kayıt tespit edildi."* -- `PARSER_ERROR` → *"Belge içeriği ayrıştırılırken hata oluştu."* - ---- - -## 6. Manual Ingestion Safety -- **File Upload (`#form-upload-file`):** Source and Document Type start with unselected disabled placeholders (`— Kaynak seçin —`, `— Belge türünü seçin —`). Form validation blocks submission and displays: *"Lütfen belgenin kaynağını seçin."* or *"Lütfen belge türünü seçin."* -- **URL Import (`#form-upload-url`):** Same safety guarantees. Accidental default classification as `Kanun` is eliminated. - ---- - -## 7. Empty States -- **Kütüphane:** Displays *"Henüz kütüphanenizde belge yok. Resmî Gazete'den veri toplayabilir veya kendi belgenizi ekleyebilirsiniz."* with CTAs **[Veri Topla]** and **[Dosya Yükle]**. -- **İnceleme (Bekleyen Kayıtlar):** Displays *"Şu anda inceleme bekleyen kayıt yok. Hazır verilerinizi dışa aktarabilir veya yeni veri toplamaya devam edebilirsiniz."* with CTAs **[Dışa Aktar]** and **[Veri Topla]**. -- **İnceleme (Sorunlar):** Displays *"Çözülmesi gereken sorun yok. Sistem şu anda kullanıcı müdahalesi gerektiren bir sorun bildirmiyor."* -- **Dışa Aktar:** If approved records count is 0, displays a prominent guidance notice (*"Henüz dışa aktarılabilecek hazır kayıt yok. Dışa aktarma yapabilmek için önce veri toplayın ve gerekiyorsa inceleme adımını tamamlayın."*) with CTAs **[Veri Topla]** and **[İncelemeye Git]**, and disables export actions with explanatory tooltips. - ---- - -## 8. Operations Contract -Frontend `loadOperationsView` calls the canonical backend endpoint `GET /api/operations/jobs` and displays `op.finished_at` (formatted with `friendlyDate`), ensuring consistent timing and status display without 404s. - ---- - -## 9. Tests Added -In `tests/acceptance/test_web_user_journey_contract.py`: -- `test_issues_contract_and_resolution`: Verifies `/api/issues` response structure, `document_title` join, and `POST /api/issues/{id}/resolve` endpoint. -- `test_blocked_record_approval_ux_contract`: Verifies that approving a record with blocking validation issues returns `BLOCKING_ISSUES_EXIST` with human-first Turkish text. -- `test_operations_jobs_contract`: Verifies `/api/operations/jobs` returns jobs with `finished_at` and expected fields. -- `test_frontend_api_contract_guard`: Expanded to statically assert registration of 24 critical frontend routes including `/api/issues`, `/api/issues/{id}/resolve`, `/api/operations/jobs`, and review actions. - ---- - -## 10. Test Results -- `uv run ruff format --check .` → **176 files already formatted (PASS)** -- `uv run ruff check .` → **All checks passed (PASS)** -- `uv run mypy src` → **Success: no issues found in 67 source files (PASS)** -- `uv run pip-audit` → **No known vulnerabilities found (PASS)** -- `uv run pytest tests/acceptance/ -ra` → **25 passed** in 42.13s (PASS) -- `uv run pytest tests/web/ tests/integration/ tests/unit/ -ra` → **202 passed** in 32.41s (PASS) -- `uv run pytest -m "not scale" -ra` → **226 passed, 4 deselected** in 15.85s (PASS) - ---- - -## 11. Zero-Knowledge Acceptance - -| Scenario | Result | Evidence | -|---|---|---| -| **First start** | PASS | Header simplified; admin token clutter removed; 5 core destinations clear. | -| **Collection** | PASS | Single-click Resmî Gazete collection with automated progress and stop. | -| **Manual upload** | PASS | Explicit dropdown selection required; accidental default classification prevented. | -| **Normal review** | PASS | Clean readable previews; 1-click approve/reject without technical IDs. | -| **Blocked review** | PASS | Clear Turkish warning with direct "Sorunları Gör" CTA to inspect the problem. | -| **Empty library** | PASS | Explanatory empty card with direct "Veri Topla" and "Dosya Yükle" CTAs. | -| **Empty review** | PASS | Positive empty state with direct "Dışa Aktar" and "Veri Topla" CTAs. | -| **Empty export** | PASS | Explains why export is not yet ready, provides CTAs, disables inactive buttons. | -| **Advanced operations** | PASS | `/api/operations/jobs` displays real jobs and correct `finished_at` timestamps. | - ---- - -## 12. Over-Engineering Check -> Was any new framework, service, worker system, scheduler, database subsystem, auth subsystem, state framework or generic workflow abstraction introduced? - -**NO** - ---- - -## 13. Remaining Usability Blockers -None. - ---- - -## 14. Final Verdict -> Can a person who knows nothing about MESA Data open the application, understand it, perform the normal workflow, and recover from common user-facing problems without documentation, terminal commands, YAML editing or internal IDs? - -**YES** - -The interface provides an intuitive 5-step journey (Ana Sayfa → Veri Topla → Kütüphane → İnceleme → Dışa Aktar), requires explicit choices for ingestion, translates all validation and lifecycle issues into clear Turkish, enables seamless navigation from issues to document contexts, and guides users with actionable next steps across all empty and error states. diff --git a/docs/UX_CLOSURE_STATE.json b/docs/UX_CLOSURE_STATE.json deleted file mode 100644 index 08610fe..0000000 --- a/docs/UX_CLOSURE_STATE.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "status": "completed", - "current_phase": "FINAL_REMEDIATION_COMPLETED", - "completed_phases": [ - "PHASE_0_REPAIR_FRONTEND_API_CONTRACTS", - "PHASE_1_ONE_COMMAND_WEB_STARTUP", - "PHASE_2_EXTRACT_HARVEST_SERVICE", - "PHASE_3_HARVEST_COLLECTION_OPERATION_JOB", - "PHASE_4_MINIMAL_HARVEST_WEB_API", - "PHASE_5_REBUILD_INFORMATION_ARCHITECTURE_AND_VIEWS", - "PHASE_6_ACCEPTANCE_TESTS_AND_JOURNEYS", - "PHASE_7_DOCUMENTATION_AND_USER_GUIDES", - "PHASE_8_REGRESSION_AND_CLEAN_VERIFICATION", - "FINAL_REMEDIATION_MUX_CERT_001_TO_004" - ], - "remediated_findings": [ - { - "id": "MUX-CERT-001", - "status": "FIXED", - "detail": "Added /releases/{release_id:path}/import-staging route alias to import_release_endpoint" - }, - { - "id": "MUX-CERT-002", - "status": "FIXED", - "detail": "Added /documents/import-url route alias to import_from_url" - }, - { - "id": "MUX-CERT-003", - "status": "FIXED", - "detail": "Corrected src_cfg.selection.allowed_document_types assignment in operations.py" - }, - { - "id": "MUX-CERT-004", - "status": "FIXED", - "detail": "Resolved all ruff lint notices and mypy type checks" - } - ], - "current_task": "FINAL_CERTIFICATION_READY", - "files_changed": [ - "src/mesa_legal_data/catalog.py", - "src/mesa_legal_data/cli.py", - "src/mesa_legal_data/operations.py", - "src/mesa_legal_data/harvest/service.py", - "src/mesa_legal_data/harvest/cli.py", - "src/mesa_legal_data/web/api.py", - "src/mesa_legal_data/web/bootstrap.py", - "src/mesa_legal_data/web/schemas.py", - "src/mesa_legal_data/web/static/index.html", - "src/mesa_legal_data/web/static/app.js", - "src/mesa_legal_data/web/static/styles.css", - "src/mesa_legal_data/web/static/ui.js", - "tests/unit/test_web_bootstrap.py", - "tests/unit/harvest/test_service_orchestration.py", - "tests/acceptance/test_web_user_journey_contract.py", - "docs/HIZLI_BASLANGIC.md", - "docs/KULLANIM_KILAVUZU.md", - "docs/UX_CLOSURE_STATE.json" - ], - "tests_run": [ - "uv run ruff check src tests (PASS: all clean)", - "uv run mypy src (PASS: 0 issues across 67 files)", - "uv run pytest tests/acceptance/test_web_user_journey_contract.py -ra (PASS: 10 passed)", - "uv run pytest -k 'not scale' -ra (PASS: 223 passed, 4 deselected)" - ], - "failures_blockers": [], - "next_exact_action": "None. System is certified ready for final release." -} diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md deleted file mode 100644 index 285c16f..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md +++ /dev/null @@ -1,28 +0,0 @@ -# Başlangıç Durumu ve Önceki Bulgular - -## Doğrulanmış başlangıç - -- Depo: `MESA_Data` (`/storage/data-law/mesa-legal-data`) -- Başlangıç SHA: `38c3fa5d3240cc944e8da611926fd220b7df145c` -- Başlangıç dalı: `main`; çalışma ağacı temizdi. -- Yetkili onarım dalı: `fix/mesa-data-legal-integrity-native-publish`. Yerel/uzak geçmişte birleştirilmemiş ve açıkça devam edilmesi gereken bir onarım dalı bulunmadığı için, talimatta izin verilen adla başlangıç SHA'sından oluşturuldu. -- Denetlenen uygulama ucu: `dfc752a386b9f1287ee42256aa4013b24309ac6c` (rapor ve durum dosyası commit'i hariç). -- Ortam: Linux, Python 3.13.12, pytest 9.1.1, AnyIO 4.15.1, Starlette 1.5.1, FastAPI 0.141.1. - -Başlangıçta `reports/` dizini yoktu; dolayısıyla bu dizin altında okunacak ya da korunacak eski rapor bulunmadı. `docs/` altındaki 11 mevcut durum/kullanım/UX belgesi okundu. Bunlar hukuki doğruluk veya canlı MESA sertifikasyonu kanıtı olarak kabul edilmedi. `docs/BUILD_STATE.json` içindeki “360 test geçti” ve `LIVE_MESA_NOT_VERIFIED` kayıtları yalnız tarihsel ipucu olarak ele alındı. - -## Önceki iddia uzlaştırma matrisi - -| İddia/kaynak | HEAD incelemesi | Kanıt | Sonuç | -|---|---|---|---| -| `docs/BUILD_STATE.json`: MVP kapanmış, testler yeşil | Test sayısı artmıştı; yeşil testler hukuki kimlik uçlarını kapsamıyordu | Ön-fix 6 yeni regresyonun 6'sı başarısız | `CONFIRMED_GAP` | -| Aynı dosya: canlı MESA doğrulanmadı | MESA çalışma kopyası, `MESA_SOURCE_ROOT` ve yetkili DEV uç noktası yok | Çapraz-depo testinin tek skip nedeni | `EXTERNAL` | -| UX kapanış raporları: panel akışları tamam | Panel/operasyon testleri mevcut ve son suite'te geçti | 413/413 test | `ALREADY_FIXED` | -| Prompt'taki tarihsel Profile B sayıları | Resmî TEST/evidence bu depoda çalıştırılmadı ve değiştirilmedi | Bu turda Profile B komutu yok | `UNVERIFIED_EVIDENCE` | -| “Plan raporu” olarak anılan `MESA_Data_Kapsamli_Denetim_Raporu.md` | Çalışma ağacında yok | `find docs`, `find reports` | `UNAVAILABLE` | - -## Başlangıçta yeniden üretilen kusurlar - -Yeni `tests/unit/test_comprehensive_legal_integrity.py` taslağındaki 6 testin tamamı onarım öncesinde kırmızıydı: mükerrer/harf ekli madde kimliği, yapısal chunk kaybı/örtüşme, chunk yasal üst kimliği, sentaktik atfın yanlış `RESOLVED` olması, edinim zamanının hukuk tarihi yapılması ve etkinlik aralığının atılması. Testler daha sonra kimlik ve zamansal dosyalara ayrıldı; son durumda tamamı yeşildir. - -İlk tam onarım-sonrası çalıştırmada 25 test, sentetik releasable fixture'ların hiçbir hukuk tarihi beyan etmemesi nedeniyle beklenen biçimde kapalı kaldı. Gate zayıflatılmadı; yalnız bu sentetik fixture'lara açık `publication_date` eklendi. İlk scale turundaki aynı nedenli 3 hata da aynı yöntemle düzeltildi. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md deleted file mode 100644 index e8876a6..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md +++ /dev/null @@ -1,34 +0,0 @@ -# Mimari ve Sözleşme Haritası - -## Gerçek veri hattı - -`resmî/manual kaynak → raw immutable dosya + SHA-256 → artifact/catalog → MIME/encoding çıkarımı → normalize canonical text → legislation/decision parser → version + article + citation JSONL → quality/privacy/validation → insan/otomatik onay → frozen release + manifest + release-index → verified publish authorization → SourceChunk → MESA v4 session/insert/mutation ledger` - -Kritik sahiplik sınırı: raw, canonical, kalite, release ve HTTP producer davranışı MESA_Data'ya aittir. MESA'nın SQL/vector/graph tüketici uygulaması ile MESA_E2E_Certification bu çalışma alanında yoktur ve değiştirilmemiştir. - -## Alan matrisi - -| Aşama | Kimlik/provenance alanları | Koruma | -|---|---|---| -| Artifact | `artifact_id`, `document_id`, `source_id`, `source_url`, `retrieved_at`, `sha256`, `raw_path`, `metadata_json` | dosya SHA ve immutable artifact collision | -| Version | `version_id`, `snapshot_date`, `effective_from/to`, `canonical_sha256`, parser/schema sürümü, onay/kalite | hukuk tarihi edinim zamanından türetilmez | -| Article | kanun ID, version ID, `article_number`, `article_kind`, exact `source_span`, etkinlik | `GEÇİCİ/EK/MÜKERRER/ordinary` ayrımı | -| Citation | source record/version, target law/article/version (doğrulanmamışsa null), exact span, status, relation hint | sentaks tek başına `RESOLVED` yapmaz | -| Release | manifest SHA, payload SHA, release-index'te version/document/record bağları | unmanifested/symlink/tamper reddi | -| Native payload | session/dataset/document/revision/chunk, content=evidence_span, revision, metadata, body idempotency | yalnız terminal `COMMITTED` başarıdır | - -## Mevcut MESA producer sözleşmesi - -- Sağlık: yapılandırılabilir, varsayılan `/health` -- Oturum: `POST /v4/sessions/start` -- Yazma: `POST /v4/memory/insert` -- Mutasyon: `GET /v4/mutations/{mutation_id}` -- Oturum kapatma: `POST /v4/sessions/{session_id}/end` -- Kimlik doğrulama: ortamdan `X-API-Key`; sır depoya/ledger'a yazılmaz. -- Kapsam: tenant/workspace/dataset/agent; hedef yapılandırma SHA ile insan onayına bağlanır. - -Bu rota ve alanlar MESA_Data producer testleriyle doğrulandı. Gerçek MESA consumer kaynağı ve DEV runtime erişilemediği için consumer uyumu `CONTRACT_ONLY` durumundadır. - -## Temsil ayrımı - -Canonical metin ve SHA release'in otoritesidir. `SourceChunk` MESA'ya taşınan türetilmiş retrieval temsilidir; içerik canonical span'dan gelir, 4.096 karakter evidence sınırına göre bölünür ve canonical dosyayı yeniden yazmaz. Embedding modeli/representation-version ve gerçek re-embed davranışı bu depoda tüketici tarafı olarak doğrulanamamıştır. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md deleted file mode 100644 index b96e838..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md +++ /dev/null @@ -1,21 +0,0 @@ -# Kusur Kaydı - -| ID | Öncelik | Yeniden üretim / kök neden | Onarım | Commit / durum | -|---|---|---|---|---| -| MD-P0-001 | P0 | `MÜKERRER MADDE` ve `MADDE 1/A` parser desenine girmiyor, kimlik çöküyordu | Parser 1.1.0 ve tür-duyarlı ID | `0a5cd5c`, kapalı | -| MD-P0-002 | P0 | Makaleler arasındaki yapısal canonical aralıklar yayın chunk'larından düşüyor; örtüşme kabul ediliyordu | Gap chunk, span sınır/örtüşme reddi, çağıran kayıtlarını mutasyonsuz işleme, parent metadata | `0a5cd5c`, kapalı | -| MD-P0-003 | P0 | `last_modified/retrieved_at`, authoritative hukuk tarihi yerine kullanılıyordu | `unknown-date` + REVIEW + açık error issue; release dışlama | `9500542`, kapalı | -| MD-P0-004 | P0 | `effective_from/to` atılıyor; gelecek sürüm aktif sürümü seçebiliyor; çelişkili bounded aralık sessizdi | Aralık saklama/doğrulama, as-of sıralama, overlap REVIEW, publish filtreleri | `9500542`, kapalı | -| MD-P0-005 | P0 | Parser'ın yalnız fallback “Madde N” üretmesi kalite PASS olabiliyordu | Tümü header-only ise BLOCK, kısmi ise REVIEW | `9500542`, kapalı | -| MD-P0-006 | P0 | Native publish job'u gerçek aktörü kaybetti; operasyon girdisinde manifest ve scope yoktu | Actor + exact manifest + target hash + frozen scope ve audit event | `17ce8f8`, kapalı | -| MD-P1-001 | P1 | Sentaktik bilinen kanun numarası corpus/version kanıtı olmadan `RESOLVED` oluyordu; harfli/türlü madde hedefleri kayıptı | `EXTRACTED`, doğru article identity; target version null kalır | `0a5cd5c`, kapalı/fail-closed | -| MD-P1-002 | P1 | Citation canonical release'te vardı ama native chunk metadata'sına taşınmıyordu | Source/target version alanları ve span-overlap citation metadata | `dfc752a`, kapalı | - -Satır örnekleri mevcut checkout içindir: parser `src/mesa_legal_data/parsers/legislation.py:31`; chunk guard `src/mesa_legal_data/publisher/chunker.py:93`; hukuk tarihi `src/mesa_legal_data/pipeline.py:223`; current selection `src/mesa_legal_data/catalog.py:244`; header-only gate `src/mesa_legal_data/quality.py:238`; native authorization `src/mesa_legal_data/web/api.py:2244`; citation handoff `src/mesa_legal_data/publisher/engine.py:338`. - -## Dış/kanıtsız kalanlar - -- EXTERNAL-P0-001: gerçek MESA tüketici rotası, schema, terminal mutation ve server-side idempotency canlı doğrulanmadı. Sahip: MESA/runtime. -- EXTERNAL-P1-001: corpus-aware citation target-version resolver ve gerçek graph projection tüketimi yok. MESA_Data yanlış çözüm üretmez; doğrulanmış target version olmadan `EXTRACTED` kalır. Sahip: MESA_Data + MESA entegrasyon turu. -- EXTERNAL-P1-002: embedding/FTS representation-version ile gerçek reindex/re-embed geçişi ölçülmedi. Sahip: MESA consumer/runtime. -- CAPACITY-P2-001: 10–20 GB hedef ölçek çalıştırılmadı. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md deleted file mode 100644 index 54ba8a0..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md +++ /dev/null @@ -1,17 +0,0 @@ -# Hukuki Kaynak, Encoding ve Kapsama - -## Doğrulananlar - -- Raw artifact SHA uyuşmazlığı pipeline/release'i durdurur; canonical satır SHA'ları release sırasında tekrar okunur. -- UTF-8 ve cp1254 fixture'ları geçti (`tests/unit/test_encoding_cp1254.py`, `tests/integration/test_pipeline_encoding.py`). Null byte/ağır replacement loss BLOCK, mojibake anomalisi REVIEW üretir. -- Sıfır makale legislation için BLOCK; düşük coverage ve büyük açıklanamayan boşluklar REVIEW/BLOCK üretir. -- Tüm madde kayıtları sadece parser fallback başlığından ibaretse PASS değildir. -- Article span'ları canonical koordinatlarında sınır ve içerik tutarlılığıyla denetlenir; publisher ayrıca out-of-range ve overlap'i reddeder. - -## Ölçülen sentetik denetim özeti - -Bu tur resmî corpus veya Profile B TEST kullanmadı. Yeni bir küçük DEV fixture setinde 5 genişletilmiş madde kimliği (`ordinary`, `EK`, `GEÇİCİ`, `MÜKERRER`, `1/A`), 1 yapısal gap örneği, 1 overlap saldırısı, 1 header-only kaynak ve 2 etkinlik aralığı senaryosu doğrulandı. Sonuçlar `source_quality_summary.csv` içindedir; bu sayılar üretim corpus toplamı değildir. - -## Destek/kısıt - -Kaynak politikaları `config/sources.yaml` ve MIME/URL güvenlik katmanıyla uygulanır. Erişilemeyen veya hukuken doğrulanmamış kaynak için başarı iddiası yoktur. Eksik orta sayfa gibi fiziksel belge bütünlüğü, kaynağın beklenen sayfa/madde sayısı bilinmiyorsa otomatik olarak ispatlanamaz; mevcut sistem coverage/gap sinyali üretir ve insan incelemesi gerekir. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md deleted file mode 100644 index b7425ef..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md +++ /dev/null @@ -1,25 +0,0 @@ -# Kanun, Madde, Chunk ve Zamansal Kimlik - -## Kimlik - -Madde parser'ı artık `MÜKERRER MADDE`, `EK MADDE`, `GEÇİCİ MADDE`, ordinary `MADDE` ve `1/A` benzeri ekli numaraları ayırır. ID örnekleri: - -- `...:article:1` -- `...:article:ek-1` -- `...:article:gecici-1` -- `...:article:mukerrer-1` -- `...:article:1-a` - -Aynı “Madde 1”, kanun ID'si parent olduğundan 6098 ve 6102 arasında çakışmaz. Chunk metadata'sı law ID/number/title, version kind/date/aralık, article ID/number/kind ve exact char koordinatını taşır. Başlık tekrar etmese bile parent kimlik korunur. - -## Span ve reconstruction - -Publisher, article span'larını sıralar; negatif/out-of-range/boş/örtüşen span'ı reddeder. Anlamlı preamble, bölüm/kısım gap'i ve annex/trailing metin ayrı chunk olur. Uzun metin 4.096 karakter MESA evidence sınırında deterministik bölünür; mevcut testler parçaların birleşiminin canonical metne eşit olduğunu doğrular. Yalnız whitespace gap'leri retrieval için ayrı junk chunk yapılmaz; hukuki anlam taşıyan gap'ler korunur. - -## Zamansal seçim - -Authoritative tarih sırası: kaynak metadata'sındaki publication/source/decision/snapshot ve beyan edilmiş effective interval. `retrieved_at` ile `last_modified` hukuki tarih yerine geçmez. Seçim rank'i bugün yürürlükte > süresi bitmiş tarihsel > gelecekte yürürlüğe girecek > tarihi bilinmeyen şeklindedir; böylece gelecek-effective sürüm aktif sürümü erken değiştirmez. - -İki farklı içerik açık ve bounded etkinlik aralıklarıyla çakışıyorsa `VERSION_EFFECTIVE_INTERVAL_AMBIGUITY` açılır, version/records pending ve kalite REVIEW yapılır. Aynı authoritative tarihte farklı içerik ve explicit supersession yoksa mevcut current seçimi korunur ve otomatik onay geri alınır. - -Kanıt: `tests/unit/test_comprehensive_legal_identity.py` 8/8; `tests/unit/test_comprehensive_temporal_integrity.py` 6/6; tarihsel chronology kontrolleri tam suite içinde geçti. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md deleted file mode 100644 index 4da7825..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md +++ /dev/null @@ -1,17 +0,0 @@ -# Release, Onay ve Değişmezlik - -Release builder yalnız current, valid, privacy-clean/approved, quality PASS, version-approved ve record-approved üyeleri seçer. Yeni temporal filtre, gelecekte başlamamış veya sona ermiş explicit aralıkları release dışında bırakır. REVIEW/BLOCK veya açık candidate-scoped error/blocker issue release'e giremez. - -## H1 eşdeğeri insan onayı - -Panel `prepare` adımında exact frozen release oluşturur ve doğrular; kullanıcıya `release_id`, `manifest_sha256`, `target_config_sha256`, hedef scope ve sayımlar döner. `publish` aynı manifest ve hedef hash'i yeniden karşılaştırır. Onarım sonrası authorization audit olayı gerçek actor, manifest, target hash, delivery/operation ID ve frozen scope'u saklar. Stale veya değiştirilmiş manifest/hedef 409 ile kapanır. - -## Değişmezlik ve negatif durumlar - -- Manifestte olmayan dosya, symlink/path traversal, payload/index hash veya count değişikliği reddedilir. -- Release-index, record → version → document üyeliğini dondurur; canlı katalog sonradan değişse bile plan frozen sürümden kurulur. -- Release revoked olduğunda devam eden delivery yeni item göndermez. -- HTTP 200/202 başarı sayılmaz; yalnız açık terminal `COMMITTED` ledger başarısıdır. -- Timeout/poll penceresi `AWAITING_MUTATION` olarak korunur; retry önce aynı remote mutation'ı uzlaştırır ve aynı idempotency key'i kullanır. - -Kanıt: `tests/integration/test_master_mvp_closure.py`, `test_release_tamper.py`, `test_release_atomicity.py`, `test_mesa_v4_publisher_e2e.py` ve final 413-test koşusu. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md deleted file mode 100644 index 19d1221..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md +++ /dev/null @@ -1,18 +0,0 @@ -# MESA Native Publish ve Idempotency - -## Producer tarafından doğrulanan sözleşme - -`MesaClient` yapılandırılabilir v4 session/insert/mutation/end rotalarını kullanır. Payload şu çekirdek bağları taşır: `session_id`, `dataset_id`, `document_id`, `revision_id`, `chunk_id`, `content`, `evidence_span`, `revision_number`, ordinal/finalize/supersedes, metadata ve body içi `idempotency_key`. API anahtarı yalnız environment'tan okunur. - -Idempotency anahtarı tenant/workspace/dataset/document/version/chunk/content hash üzerinden deterministiktir. Ledger item gönderilmeden önce yazılır. Yanıt kaybı ve crash-window testleri aynı key ve varsa aynı mutation ID ile uzlaştırır; cross-release aynı mantıksal chunk `SKIPPED` olur. - -Durum normalizasyonu yalnız bilinen açık state'leri kabul eder. 200/201/202 içinde state veya gerekli mutation ID yoksa FAIL'dir. `QUEUED/PROCESSING` poll edilir; pencere biterse `AWAITING_MUTATION`; `COMMITTED`, `FAILED`, `REJECTED` terminaldir. 401/403, 429/503, bilinmeyen endpoint/state ve target güvenlik kontrolleri fail-closed test edilmiştir. - -## Kanıt düzeyi - -- Mock/stateful adapter: doğrulandı (`tests/integration/test_mesa_v4_publisher_e2e.py`, 3/3; post-master publisher kontrolleri). -- Frozen release, actor/manifest/target/scope bağlama: doğrulandı. -- Gerçek MESA source/runtime: yok. -- Canlı mutation ID, gerçek SQL/vector/graph projection ve restart-sonrası server idempotency: `UNVERIFIED_ENV`. - -Sonuç: `NATIVE_MESA_INTEGRATION=CONTRACT_ONLY`. Mock başarı canlı entegrasyon başarısı olarak raporlanmamıştır. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md deleted file mode 100644 index 5c37e77..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md +++ /dev/null @@ -1,15 +0,0 @@ -# İlişkisel Atıflar ve Temsil - -## Atıf doğruluğu - -Parser numbered law ve alias referanslarını exact Unicode span ile çıkarır. `amends`, `repeals`, `adds` fiilleri relation hint üretir. `1/A` article suffix olarak korunur; `27/2` ve `25/II` article 27/25 içindeki alt bölüm olarak parent article ID'ye bağlanır. `EK/GEÇİCİ/MÜKERRER` türleri farklı hedef kimliği üretir. - -Sentaktik hedef hiçbir zaman corpus varlığı veya doğru tarihsel version kanıtı değildir. Bu nedenle bilinen kanunlar da `EXTRACTED`, belirsiz genel atıflar `UNRESOLVED` kalır. Yeni kayıtlar `source_version_id` ve nullable `target_version_id` taşır; doğrulanmamış target version uydurulmaz. - -Native plan artık release-owned citation kaydını, exact source span ile örtüşen chunk'ın metadata'sına taşır. Alanlar citation/source record/source version/target law/article/version/status/relation/span ve `fully_contained` bilgisidir. Publisher metinden yeniden graph edge tahmin etmez. - -## Kalan tüketici bağımlılığı - -MESA graph projection kodu yoktur. Metadata'nın gerçek graph edge'e dönüştürülmesi, target version corpus resolver'ı ve version-as-of graph sorgusu canlı consumer ile doğrulanmalıdır. Bu nedenle kabul kapısı G producer tarafında fail-closed `VERIFIED_PASS`, gerçek graph consumption için `UNVERIFIED_ENV` olarak ayrılmıştır. - -Canonical metin immutable kalır; retrieval chunk ve gelecekteki embedding/FTS representation türetilmiştir. Bu depoda embedding model/version alanının tüketici tarafından nasıl kullanıldığı görülemediğinden gerçek reindex/re-embed `EMBEDDING_DEPENDENCY_BLOCKED` durumundadır. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md deleted file mode 100644 index 964e500..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md +++ /dev/null @@ -1,63 +0,0 @@ -# Regresyon ve Adversarial Sonuçlar - -## Son koşular - -| Komut | Sonuç | -|---|---| -| `.venv/bin/pytest -ra --durations=20 --tb=short` | 413 passed, 1 skipped, 78.95 s | -| `.venv/bin/pytest -m "not scale" -ra --durations=20 --tb=short` | 409 passed, 1 skipped, 4 deselected, 33.45 s | -| `/usr/bin/time -v .venv/bin/pytest -m scale ...` | 4 passed, 1 skipped, 409 deselected, 46.53 s, peak RSS 171,784 KiB | -| `.venv/bin/ruff format --check .` | 217 files formatted | -| `.venv/bin/ruff check .` | PASS | -| `.venv/bin/mypy src` | 78 source files, 0 issue | -| `.venv/bin/pip-audit` | bilinen açık yok; yerel paket PyPI'da olmadığı için skip | -| `git diff --check` | PASS | - -Tek skip: `tests/integration/test_mesa_v4_vector_cross_repo.py`; `MESA_SOURCE_ROOT` ayarlı değil ve sibling MESA checkout bulunmadı. TestClient koşuları managed filesystem sandbox dışında çalıştırıldı; sandbox AF_UNIX wakeup yazmasını EPERM ile engelliyordu. Aynı minimal probe dışarıda 200 döndü; bağımlılık lock'u değiştirilmedi. - -## 38 maddelik negatif matris - -| # | Sonuç | Kanıt/Not | -|---|---|---| -| 1 | PASS | raw/canonical SHA ve tamper testleri | -| 2 | PASS | zero article/coverage/header-only gate | -| 3 | PASS | cp1254/UTF-8 testleri | -| 4 | PASS | mojibake/replacement quality testleri | -| 5 | PASS | parent law ID dahil article ID | -| 6 | PASS | header-only BLOCK | -| 7 | PASS | chunk parent metadata testi | -| 8 | PASS | ordinary/EK/GEÇİCİ/MÜKERRER ayrımı | -| 9 | PASS | overlap/mixed span redde kapalı | -| 10 | PASS | long split/reconstruction | -| 11 | PASS | overlap duplicate üretmez | -| 12 | PASS | fetched_at hukuk tarihi değildir | -| 13 | PASS | out-of-order historical import | -| 14 | PASS | effective interval/current filtreleri | -| 15 | PASS | bounded overlap REVIEW | -| 16 | PASS | content hash/version/manifest kimliği | -| 17 | PASS | deterministik hash/chunk/release | -| 18 | PASS | approval/quality/release seçimi | -| 19 | PASS | manifest+target+actor+scope yeniden doğrulama | -| 20 | PASS | manifest tamper/unmanifested/order bağları | -| 21 | PASS | schema/response validation | -| 22 | PASS | unknown route/contract FAIL | -| 23 | PASS | 200/202 tek başına COMMITTED değil | -| 24 | PASS | AWAITING_MUTATION reconcile | -| 25 | CONTRACT_ONLY | body idempotency producer testi; gerçek consumer yok | -| 26 | PASS | cross-release dedup/repeat publish | -| 27 | PASS | response-loss/ledger retry | -| 28 | PASS | auth ve target-scope fail-closed | -| 29 | PASS | finite retry/poll | -| 30 | PASS | ambiguous citation UNRESOLVED | -| 31 | PASS | target version doğrulanmadan RESOLVED değil | -| 32 | PASS | out-of-range span redde kapalı | -| 33 | PASS | chunking canonical/raw'u değiştirmez | -| 34 | UNVERIFIED_ENV | consumer representation version/re-embed yok | -| 35 | PASS | rapor canlı başarı iddia etmiyor | -| 36 | PASS | yalnız sentetik DEV fixture; resmî TEST çalıştırılmadı | -| 37 | PASS | release path/symlink güvenlik testleri | -| 38 | PASS | pipeline/release atomicity ve rollback | - -## Önce/sonra - -İlk hedefli koşu 6/6 kırmızıydı; ilgili hedefli son koşu 27/27 yeşil oldu. İlk full onarım koşusundaki 25 hata ve ilk scale koşusundaki 3 hata, tarih beyan etmeyen sentetik release fixture'larını gösterdi; gate düşürülmeden fixture metadata'sı düzeltildi. Nihai full koşu 413/413 geçti. Resmî Recall@5/MRR karşılaştırması yapılmadı. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md deleted file mode 100644 index 9d499ff..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md +++ /dev/null @@ -1,9 +0,0 @@ -# Ölçek ve Operasyon Hazırlığı - -Scale işaretli 4 testin tamamı geçti. Ölçülen birleşik koşu 46.53 saniye duvar zamanı, 171,784 KiB maksimum RSS ve 735,504 filesystem output block kullandı. En yavaş testler: 10.000 article hardening 21.44 s, bulk review streaming 13.55 s, operations scale 10.35 s. - -Mevcut mimari canonical JSONL'yi streaming işler; release seçimini geçici SQLite spool ile yapar ve canonical part dosyasını tek geçişte okuyan testlere sahiptir. Approval streaming, bounded fetchmany ve operasyon recovery testleri geçti. Immutable canonical core'dan derived staging/projection yeniden kurulabilir. - -Bu ölçüm küçük sentetik ölçektir. 10–20 GB artifact/canonical corpus, gerçek disk baskısı, backup/restore süresi, MESA vector/graph rebuild ve eşzamanlı operasyon davranışı çalıştırılmadı. Sonuç: `CAPACITY_NOT_PROVEN_AT_TARGET_SCALE`. - -Önerilen sonraki ölçüm, resmî TEST'ten bağımsız sabit DEV corpus ile 1x/5x/20x koşularında wall time, peak RSS, disk amplification, release spool boyutu ve restart recovery ölçmektir. Üretim verisi üzerinde destructive stress önerilmez. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md deleted file mode 100644 index 3175b09..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md +++ /dev/null @@ -1,24 +0,0 @@ -# Çapraz Depo Teslimi - -## Erişim durumu - -`/storage` altında sibling `MESA` veya `MESA_E2E_Certification` checkout bulunmadı; `MESA_SOURCE_ROOT` ayarlı değildi. Bu iki depo read-only bile incelenemedi ve hiçbir dosyaları değiştirilmedi. Resmî Profile B evidence/TEST verisine dokunulmadı. - -## MESA sahibine doğrulatılacak sözleşme - -1. Session start request: tenant/workspace, `dataset_ids`, agent; 201 ve nonempty `session_id`. -2. Memory insert: document/revision/chunk identity, source_ref, content/evidence_span ≤4096, revision/finalize/supersedes, metadata, body `idempotency_key`. -3. Mutation response: açık state; queued/processing ise nonempty mutation ID; status route ve session yaşam döngüsü. -4. Server-side duplicate key semantiği, response-loss/restart sonrası aynı mutation'a dönüş. -5. Citation metadata'nın graph projection'a typed/versioned edge olarak nasıl alındığı; `EXTRACTED`/null target version'ın graph'a kesin edge yapılmaması. -6. Representation/model version değişiminde gerçek reindex/re-embed ve canonical provenance korunumu. - -## E2E sahibine önerilen takip - -- Yeni producer branch SHA'sını MESA ile aynı DEV scope'ta checkout edin. -- İzole DEV release'i native akışla yayınlayın; gerçek mutation ID'leri terminale kadar kaydedin. -- SQL canonical, vector metadata ve graph edge'lerinde document/version/article/span/hash provenance'ını kontrol edin. -- Aynı release'i tekrar ve process restart sonrası gönderip logical duplicate oluşmadığını kanıtlayın. -- Bu sonuçları yeni bir DEV evidence run olarak saklayın; mevcut Profile B TEST sonucunu yeniden etiketlemeyin. - -Çapraz-depo testi, `MESA_SOURCE_ROOT` sağlandığında çalıştırılmaya hazırdır; bu turdaki tek skip budur. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md deleted file mode 100644 index 5e255e5..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md +++ /dev/null @@ -1,30 +0,0 @@ -# Nihai Hazırlık Kararı - -## Karar - -`MESA_DATA_REPAIR_VERIFIED_PUSHED`: tüm doğrulanmış MESA_Data-sahipli P0/P1 kusurları kapatılmış, yerel doğrulamaları geçmiş ve yetkili `fix/mesa-data-legal-integrity-native-publish` dalına gönderilmiştir. Commit kimliği kendi içeriğinde öz-referanslı olamayacağı için kesin terminal SHA teslim mesajında verilir. Bu karar `PROFILE_B_PASS` veya `PRODUCTION_READY` değildir. - -Denetlenen uygulama SHA: `dfc752a386b9f1287ee42256aa4013b24309ac6c`. - -## Kabul kapıları - -| Kapı | Sınıf | Kanıt | -|---|---|---| -| A raw→canonical izlenebilirlik | VERIFIED_PASS | SHA, source/provenance, release-index/tamper testleri | -| B doğru law/article/chunk/version | VERIFIED_PASS | yeni identity/chunker testleri | -| C current/historical zaman | VERIFIED_PASS | unknown/future/out-of-order/overlap testleri | -| D gate bypass yok | VERIFIED_PASS | quality/approval/release/adversarial suite | -| E gerçek MESA API uyumu | UNVERIFIED_ENV | producer contract testli; consumer checkout/runtime yok | -| F idempotency | VERIFIED_PASS | stateful mock/crash/replay testleri; canlı DEV kısmı UNVERIFIED_ENV | -| G citation/graph | VERIFIED_PASS | producer typed/versioned/span evidence ve fail-closed status; gerçek graph consumer UNVERIFIED_ENV | -| H canonical/retrieval ayrımı | VERIFIED_PASS | raw/canonical immutable, chunk türetilmiş; gerçek re-embed UNVERIFIED_ENV | -| I negatif/regresyon | VERIFIED_PASS | 413 passed, 38 maddelik matris | -| J kalan bağımlılıklar açık | VERIFIED_PASS | MESA/E2E/embedding/corpus/scale açıkça listeli | - -## Kalan engeller - -MESA_Data içinde doğrulanmış açık P0/P1 yoktur. Dış engeller: gerçek MESA DEV runtime ve source yok; gerçek graph/vector/SQL tüketimi yok; embedding provider yok; resmî corpus/TEST yetkili çalıştırması yok; 10–20 GB kapasite ölçülmedi. Güvenli sonraki adım `10_CROSS_REPO_HANDOFF.md` içindeki bounded DEV entegrasyonudur. - -`PROFILE_B_CERTIFICATION: NOT_RUN_NOT_CLAIMED` - -`CAPACITY_10_20_GB: NOT_PROVEN` diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv deleted file mode 100644 index 6f5630c..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv +++ /dev/null @@ -1,13 +0,0 @@ -stage,producer_field,consumer_field,verification,provenance_hash -raw,artifacts.sha256,canonical.source.artifact_sha256,PASS,SHA-256 -raw,artifacts.raw_path,canonical.source.artifact_path,PASS,SHA-256 -version,versions.version_id,canonical.version.version_id,PASS,canonical_sha256 -article,article.id,chunk.metadata.article_id,PASS,content_hash -article,article.article_kind,chunk.metadata.article_kind,PASS,content_hash -article,article.source_span,chunk.char_start/char_end,PASS,content_hash -temporal,version.effective_from/to,chunk.metadata.effective_from/to,PASS,manifest_sha256 -citation,citation.source_version_id,chunk.metadata.citations[].source_version_id,PASS,manifest_sha256 -citation,citation.target_version_id,chunk.metadata.citations[].target_version_id,PASS_NULL_UNTIL_VERIFIED,manifest_sha256 -release,release-index.version/document/record,publisher frozen membership,PASS,manifest_sha256 -payload,chunk.content_hash,metadata.mesa_data_content_hash,PASS,idempotency_key -payload,release.manifest_sha256,delivery.release_manifest_sha256,PASS,manifest_sha256 diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json deleted file mode 100644 index eded1e9..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json +++ /dev/null @@ -1,74 +0,0 @@ -[ - { - "id": "MD-P0-001", - "severity": "P0", - "owner": "MESA_Data", - "proof": "tests/unit/test_comprehensive_legal_identity.py::test_article_parser_preserves_extended_turkish_article_identity", - "fix_sha": "0a5cd5c", - "verification_state": "VERIFIED_FIXED" - }, - { - "id": "MD-P0-002", - "severity": "P0", - "owner": "MESA_Data", - "proof": "tests/unit/test_comprehensive_legal_identity.py::test_chunk_plan_is_lossless_and_rejects_overlapping_article_identity", - "fix_sha": "0a5cd5c", - "verification_state": "VERIFIED_FIXED" - }, - { - "id": "MD-P0-003", - "severity": "P0", - "owner": "MESA_Data", - "proof": "tests/unit/test_comprehensive_temporal_integrity.py::test_acquisition_time_is_not_used_as_legal_snapshot_date", - "fix_sha": "9500542", - "verification_state": "VERIFIED_FIXED" - }, - { - "id": "MD-P0-004", - "severity": "P0", - "owner": "MESA_Data", - "proof": "tests/unit/test_comprehensive_temporal_integrity.py future/overlap/invalid interval tests", - "fix_sha": "9500542", - "verification_state": "VERIFIED_FIXED" - }, - { - "id": "MD-P0-005", - "severity": "P0", - "owner": "MESA_Data", - "proof": "tests/unit/test_comprehensive_temporal_integrity.py::test_header_only_legislation_cannot_pass_quality_gate", - "fix_sha": "9500542", - "verification_state": "VERIFIED_FIXED" - }, - { - "id": "MD-P0-006", - "severity": "P0", - "owner": "MESA_Data", - "proof": "tests/integration/test_master_mvp_closure.py::test_final_panel_confirmation_binds_exact_release_manifest_and_target", - "fix_sha": "17ce8f8", - "verification_state": "VERIFIED_FIXED" - }, - { - "id": "MD-P1-001", - "severity": "P1", - "owner": "MESA_Data", - "proof": "tests/unit/test_comprehensive_legal_identity.py citation cases", - "fix_sha": "0a5cd5c", - "verification_state": "VERIFIED_FIXED_FAIL_CLOSED" - }, - { - "id": "MD-P1-002", - "severity": "P1", - "owner": "MESA_Data", - "proof": "tests/integration/test_master_mvp_closure.py native citation metadata assertion", - "fix_sha": "dfc752a", - "verification_state": "VERIFIED_FIXED_PRODUCER" - }, - { - "id": "EXTERNAL-P0-001", - "severity": "P0_EXTERNAL", - "owner": "MESA/runtime", - "proof": "MESA_SOURCE_ROOT unset; cross-repo test skipped", - "fix_sha": null, - "verification_state": "UNVERIFIED_ENV" - } -] diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv deleted file mode 100644 index 158f446..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv +++ /dev/null @@ -1,15 +0,0 @@ -producer_field,consumer_field,expected,observed,status -session.tenant_id,session tenant scope,nonempty,target settings emitted,CONTRACT_ONLY -session.workspace_id,session workspace scope,nonempty,target settings emitted,CONTRACT_ONLY -session.dataset_ids,session dataset scope,one configured dataset,list emitted,CONTRACT_ONLY -insert.document_id,canonical document identity,stable law/decision ID,emitted and mock asserted,PASS_MOCK -insert.revision_id,legal version identity,exact frozen version,emitted and mock asserted,PASS_MOCK -insert.chunk_id,chunk identity,deterministic within version,emitted and replay tested,PASS_MOCK -insert.content/evidence_span,retrieval evidence,identical canonical slice <=4096,validated,PASS_MOCK -insert.metadata.article_id,article identity,parent law plus kind/number,validated,PASS_MOCK -insert.metadata.citations,graph evidence,typed span/version candidate,validated producer side,PASS_MOCK -insert.idempotency_key,server idempotency,stable body field,stable key tested consumer unknown,CONTRACT_ONLY -response.mutation_id,remote mutation identity,required for nonterminal,fail closed if absent,PASS_MOCK -response.state,ledger terminal state,explicit COMMITTED only,normalization tested,PASS_MOCK -GET /v4/mutations/{id},reconciliation,terminal poll after async/timeout,stateful mock tested,CONTRACT_ONLY -server projections,SQL/vector/graph provenance,document/version/chunk/span/hash,no runtime available,UNVERIFIED_ENV diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv deleted file mode 100644 index d350ddf..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv +++ /dev/null @@ -1,10 +0,0 @@ -fixture,source_type,units_expected,units_observed,quality_result,warning_or_blocker,scope -extended_article_identity,synthetic_text,5,5,PASS,,DEV_ONLY -structural_gap,synthetic_text,1_gap,1_gap,PASS,,DEV_ONLY -overlapping_spans,synthetic_text,REJECT,REJECT,BLOCK,span_overlap,DEV_ONLY -missing_legal_date,synthetic_html,REVIEW,REVIEW,REVIEW,LEGAL_DATE_MISSING,DEV_ONLY -invalid_effective_interval,synthetic_html,REVIEW,REVIEW,REVIEW,EFFECTIVE_INTERVAL_INVALID,DEV_ONLY -overlapping_effective_intervals,synthetic_html,2_review,2_review,REVIEW,VERSION_EFFECTIVE_INTERVAL_AMBIGUITY,DEV_ONLY -header_only_legislation,synthetic_html,BLOCK,BLOCK,BLOCK,article_body_presence,DEV_ONLY -cp1254_encoding,synthetic_bytes,2_tests,2_pass,PASS,,DEV_ONLY -official_profile_b,official_test,NOT_RUN,NOT_RUN,UNVERIFIED,protected evidence not accessed,EXTERNAL diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json deleted file mode 100644 index 4923247..0000000 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json +++ /dev/null @@ -1,60 +0,0 @@ -[ - { - "command": ".venv/bin/pytest tests/unit/test_comprehensive_legal_integrity.py -q", - "exit_code": 1, - "executed": 6, - "passed": 0, - "failed": 6, - "skipped": 0, - "evidence": "pre-fix reproduction; tests later split into comprehensive_legal_identity and comprehensive_temporal_integrity" - }, - { - "command": ".venv/bin/pytest -m \"not scale\" -ra --durations=20 --tb=short", - "exit_code": 0, - "executed": 409, - "passed": 409, - "failed": 0, - "skipped": 1, - "deselected": 4, - "evidence": "terminal run 2026-10-11; cross-repo skip only" - }, - { - "command": "/usr/bin/time -v .venv/bin/pytest -m scale -ra --durations=20 --tb=short", - "exit_code": 0, - "executed": 4, - "passed": 4, - "failed": 0, - "skipped": 1, - "deselected": 409, - "wall_seconds": 46.53, - "peak_rss_kib": 171784, - "evidence": "bounded synthetic scale; not 10-20 GB" - }, - { - "command": ".venv/bin/pytest -ra --durations=20 --tb=short", - "exit_code": 0, - "executed": 413, - "passed": 413, - "failed": 0, - "skipped": 1, - "evidence": "final full suite; skip requires MESA_SOURCE_ROOT" - }, - { - "command": ".venv/bin/ruff format --check . && .venv/bin/ruff check . && .venv/bin/mypy src", - "exit_code": 0, - "executed": 3, - "passed": 3, - "failed": 0, - "skipped": 0, - "evidence": "217 formatted files; lint pass; 78 typed source files" - }, - { - "command": ".venv/bin/pip-audit", - "exit_code": 0, - "executed": 1, - "passed": 1, - "failed": 0, - "skipped": 1, - "evidence": "no known vulnerabilities; local mesa-legal-data package not on PyPI" - } -] diff --git a/src/mesa_legal_data/cli.py b/src/mesa_legal_data/cli.py index 13ff772..147a4d5 100644 --- a/src/mesa_legal_data/cli.py +++ b/src/mesa_legal_data/cli.py @@ -8,8 +8,12 @@ app = typer.Typer(help="MESA Legal Data CLI") collect_app = typer.Typer(help="Collect legal data artifacts from sources.") +source_app = typer.Typer(help="Manage source operational settings and parser certifications.") +publish_app = typer.Typer(help="Configure MESA and publish a human-confirmed verified release.") app.add_typer(collect_app, name="collect") app.add_typer(harvest_app, name="harvest") +app.add_typer(source_app, name="source") +app.add_typer(publish_app, name="publish") @app.command() @@ -194,7 +198,7 @@ def collect_url( def collect_seed( fixtures_dir: Path | None = typer.Option(None, "--fixtures-dir", help="Optional local directory with PDF fixtures"), ): - """Imports the 12 core seed legislation items.""" + """Imports core seed legislation items.""" from mesa_legal_data.collectors.seed import run_seed_collection try: @@ -208,6 +212,28 @@ def collect_seed( raise typer.Exit(code=1) +@collect_app.command("bedesten") +def collect_bedesten_cli( + unit: str = typer.Option(..., "--unit", help="HGK, CGK or IBK"), + from_date: str = typer.Option(..., "--from", help="First decision date (YYYY-MM-DD, from 2021-01-01)"), + to_date: str | None = typer.Option(None, "--to", help="Last decision date (YYYY-MM-DD)"), + limit: int = typer.Option(10, "--limit", min=1), +): + """Collect scoped Yargıtay decisions and run the pipeline.""" + from datetime import date + + from mesa_legal_data.sources.bedesten import collect_bedesten + + try: + artifacts = collect_bedesten( + unit, date.fromisoformat(from_date), date.fromisoformat(to_date) if to_date else None, limit=limit + ) + typer.echo(f"Collected {len(artifacts)} Bedesten decisions.") + except Exception as exc: + typer.secho(f"Bedesten collection failed: {exc}", fg=typer.colors.RED) + raise typer.Exit(code=1) + + @app.command() def report(): """Generates quality and catalog summary report.""" @@ -642,5 +668,292 @@ def sync_cmd( ) +@source_app.command("show") +def source_show( + source_id: str = typer.Argument(..., help="Source identifier, e.g. resmi_gazete, mevzuat, aym"), +): + """Shows operational settings and parser certifications for a source.""" + from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + get_source_operational_settings, + list_parser_certifications, + ) + + db_path = get_db_path() + if not db_path.exists(): + typer.secho(f"Catalog database not found at {db_path}", fg=typer.colors.RED) + raise typer.Exit(code=1) + + conn = get_connection() + try: + settings = get_source_operational_settings(conn, source_id) + certs = list_parser_certifications(conn, source_id=source_id) + + typer.secho(f"=== Source Operational Settings: {source_id} ===", fg=typer.colors.CYAN, bold=True) + typer.echo(f"Enabled : {settings['enabled']}") + typer.echo(f"Auto-approval Enabled : {settings['auto_approval_enabled']}") + typer.echo(f"Weekly Sample Count : {settings['weekly_sample_count']}") + typer.echo(f"Updated At : {settings.get('updated_at', '-')}") + typer.echo(f"Updated By : {settings.get('updated_by', '-')}") + + typer.secho("\n=== Parser Certifications ===", fg=typer.colors.CYAN, bold=True) + if not certs: + typer.echo("No parser certifications found.") + else: + for cert in certs: + status_color = typer.colors.GREEN if cert["certified"] else typer.colors.RED + status_text = "CERTIFIED" if cert["certified"] else "REVOKED" + typer.echo( + f"- {cert['parser_name']} v{cert['parser_version']}: " + + typer.style(status_text, fg=status_color, bold=True) + + f" (by {cert.get('certified_by', '-')}, at {cert.get('certified_at', '-')})" + ) + finally: + conn.close() + + +@source_app.command("auto-approval") +def source_auto_approval( + source_id: str = typer.Argument(..., help="Source identifier, e.g. resmi_gazete, mevzuat"), + enable: bool = typer.Option(..., "--enable/--disable", help="Enable or disable auto-approval"), + by: str = typer.Option("operator", "--by", help="Operator name or identifier"), +): + """Enables or disables auto-approval for a source.""" + from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + get_source_operational_settings, + upsert_source_operational_settings, + ) + + db_path = get_db_path() + if not db_path.exists(): + typer.secho(f"Catalog database not found at {db_path}", fg=typer.colors.RED) + raise typer.Exit(code=1) + + conn = get_connection() + try: + current = get_source_operational_settings(conn, source_id) + upsert_source_operational_settings( + conn, + source_id=source_id, + enabled=current["enabled"], + auto_approval_enabled=enable, + weekly_sample_count=current["weekly_sample_count"], + updated_by=by, + ) + action = "enabled" if enable else "disabled" + typer.secho( + f"Auto-approval {action} for source '{source_id}' by {by}.", + fg=typer.colors.GREEN, + ) + finally: + conn.close() + + +@source_app.command("certify-parser") +def source_certify_parser( + source_id: str = typer.Argument(..., help="Source identifier, e.g. resmi_gazete, mevzuat"), + parser: str = typer.Option(..., "--parser", help="Parser name"), + version: str = typer.Option(..., "--version", help="Parser version"), + by: str = typer.Option("operator", "--by", help="Operator name or identifier"), + revoke: bool = typer.Option(False, "--revoke", help="Revoke certification instead of certifying"), +): + """Certifies or revokes certification for a parser version for a source.""" + from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + set_parser_certification, + ) + + db_path = get_db_path() + if not db_path.exists(): + typer.secho(f"Catalog database not found at {db_path}", fg=typer.colors.RED) + raise typer.Exit(code=1) + + conn = get_connection() + try: + set_parser_certification( + conn, + source_id=source_id, + parser_name=parser, + parser_version=version, + certified=not revoke, + certified_by=by, + ) + action = "revoked" if revoke else "certified" + color = typer.colors.YELLOW if revoke else typer.colors.GREEN + typer.secho( + f"Parser '{parser}' v{version} {action} for source '{source_id}' by {by}.", + fg=color, + ) + finally: + conn.close() + + +@publish_app.command("target") +def publish_target( + base_url: str = typer.Option(..., "--base-url"), + tenant_id: str = typer.Option(..., "--tenant-id"), + workspace_id: str = typer.Option(..., "--workspace-id"), + dataset_id: str = typer.Option(..., "--dataset-id"), + agent_id: str = typer.Option(..., "--agent-id"), + target_key: str = typer.Option("default", "--target-key"), + content_limit_chars: int = typer.Option(32768, "--content-limit-chars", min=1024, max=32768), + health_path: str = typer.Option("/health", "--health-path"), + session_start_path: str = typer.Option("/v4/sessions/start", "--session-start-path"), + publish_path: str = typer.Option("/v4/memory/insert", "--publish-path"), + mutation_status_path_template: str = typer.Option("/v4/mutations/{mutation_id}", "--mutation-status-path-template"), + session_end_path_template: str = typer.Option("/v4/sessions/{session_id}/end", "--session-end-path-template"), +): + """Save the non-secret MESA target and print its confirmation hash.""" + from mesa_legal_data.catalog import get_connection + from mesa_legal_data.publisher.engine import target_config_sha256 + from mesa_legal_data.publisher.ledger import upsert_mesa_target_settings + from mesa_legal_data.publisher.models import MesaTargetSettings + from mesa_legal_data.web.schemas import MesaTargetSettingsUpdateRequest + + try: + request = MesaTargetSettingsUpdateRequest( + base_url=base_url, + tenant_id=tenant_id, + workspace_id=workspace_id, + dataset_id=dataset_id, + agent_id=agent_id, + content_limit_chars=content_limit_chars, + health_path=health_path, + session_start_path=session_start_path, + publish_path=publish_path, + mutation_status_path_template=mutation_status_path_template, + session_end_path_template=session_end_path_template, + ) + settings = MesaTargetSettings( + target_key=target_key, + **request.model_dump(), + contract_source=( + "configured" + if request.health_path + and request.session_start_path + and request.publish_path + and "{mutation_id}" in request.mutation_status_path_template + else "unknown" + ), + ) + conn = get_connection() + try: + upsert_mesa_target_settings(conn, settings) + finally: + conn.close() + typer.echo(f"Target config SHA-256: {target_config_sha256(settings)}") + except Exception as exc: + typer.secho(f"Target configuration failed: {exc}", fg=typer.colors.RED) + raise typer.Exit(code=1) + + +@publish_app.command("run") +def publish_run( + release_id: str = typer.Option(..., "--release-id"), + confirm_manifest: str = typer.Option(..., "--confirm-manifest"), + confirm_target: str = typer.Option(..., "--confirm-target"), + target_key: str = typer.Option("default", "--target-key"), + by: str = typer.Option("cli", "--by", help="Human authorizing this delivery"), +): + """Publish the confirmed immutable release synchronously; succeed only on COMMITTED.""" + import json + import uuid + + from mesa_legal_data.audit import log_audit_event + from mesa_legal_data.catalog import get_connection, get_release + from mesa_legal_data.publisher.client import MesaClient + from mesa_legal_data.publisher.engine import build_delivery_plan, execute_publish_delivery, target_config_sha256 + from mesa_legal_data.publisher.ledger import get_mesa_target_settings + from mesa_legal_data.web.schemas import MesaPublishRequest + + try: + request = MesaPublishRequest( + release_id=release_id, + manifest_sha256=confirm_manifest, + target_config_sha256=confirm_target, + target_key=target_key, + ) + conn = get_connection() + try: + settings = get_mesa_target_settings(conn, target_key) + release = get_release(conn, release_id) + if not release: + raise ValueError(f"RELEASE_NOT_FOUND: {release_id}") + if release["status"] not in ("verified", "published"): + raise ValueError(f"RELEASE_STATUS_INVALID: {release['status']}") + if release["manifest_sha256"] != request.manifest_sha256: + raise ValueError("RELEASE_CONFIRMATION_MISMATCH: confirmed manifest does not match") + if target_config_sha256(settings) != request.target_config_sha256: + raise ValueError("TARGET_CONFIRMATION_MISMATCH: MESA target changed after confirmation") + _, summary = build_delivery_plan(target_key=target_key, release_id=release_id) + report = MesaClient(settings=settings).run_preflight_checks( + ready_documents_count=summary.ready_documents, + ready_versions_count=summary.ready_versions, + estimated_chunks_count=summary.estimated_chunks, + total_canonical_bytes=summary.total_canonical_bytes, + blocked_versions_count=summary.blocked_versions_excluded, + unreadable_versions_count=summary.unreadable_versions_excluded, + ) + if report.overall_status == "FAIL": + raise ValueError( + "PREFLIGHT_FAILED: " + "; ".join(c.message for c in report.checks if c.status == "FAIL") + ) + delivery_id = f"del-{uuid.uuid4().hex[:12]}" + log_audit_event( + conn, + actor=by, + action="mesa_publish_authorize", + subject_type="release", + subject_id=release_id, + new_sha256=request.manifest_sha256, + details_json=json.dumps( + { + "delivery_id": delivery_id, + "target_key": target_key, + "target_config_sha256": request.target_config_sha256, + "scope": summary.model_dump(), + }, + sort_keys=True, + ), + ) + finally: + conn.close() + result = execute_publish_delivery( + delivery_id=delivery_id, + release_id=release_id, + target_key=target_key, + expected_target_config_sha256=request.target_config_sha256, + ) + except Exception as exc: + typer.secho(f"Publish failed: {exc}", fg=typer.colors.RED) + raise typer.Exit(code=1) + typer.echo(f"Delivery {result['delivery_id']}: {result['status']}") + if result["status"] != "COMMITTED": + raise typer.Exit(code=1) + + +@publish_app.command("status") +def publish_status(delivery_id: str): + """Print a delivery's persisted status and counters.""" + import json + + from mesa_legal_data.catalog import get_connection + from mesa_legal_data.publisher.ledger import get_delivery + + conn = get_connection() + try: + delivery = get_delivery(conn, delivery_id) + finally: + conn.close() + if not delivery: + typer.secho(f"Delivery not found: {delivery_id}", fg=typer.colors.RED) + raise typer.Exit(code=1) + typer.echo(json.dumps(delivery, indent=2, ensure_ascii=False)) + + if __name__ == "__main__": app() diff --git a/src/mesa_legal_data/collectors/seed.py b/src/mesa_legal_data/collectors/seed.py index e3a1585..91f0d26 100644 --- a/src/mesa_legal_data/collectors/seed.py +++ b/src/mesa_legal_data/collectors/seed.py @@ -1,8 +1,11 @@ +from datetime import UTC, datetime from pathlib import Path import yaml # type: ignore[import-untyped] +from mesa_legal_data.catalog import get_connection, get_document from mesa_legal_data.models import FetchedArtifact +from mesa_legal_data.pipeline import process_artifact_pipeline from mesa_legal_data.sources import import_manual_file, import_manual_url @@ -26,10 +29,17 @@ def run_seed_collection( """ Collects seed legislation. If local_fixtures_dir is provided, imports from local files; otherwise imports via URL. Continues on individual item failures and reports them. + Runs process_artifact_pipeline for every artifact that is new in this run. + Prints, per item: number, title, collected / unchanged / failed, pipeline status. """ items = load_seed_config(config_path) - collected = [] - skipped: list[tuple[str, str, str]] = [] # (number, title, reason) + collected: list[FetchedArtifact] = [] + + today_utc = datetime.now(UTC).strftime("%Y-%m-%d") + extra_meta = { + "snapshot_date": today_utc, + "source_role": "consolidated_text", + } for item in items: doc_id = item["document_id"] @@ -40,6 +50,14 @@ def run_seed_collection( source_id = "mevzuat" try: + conn = get_connection() + try: + document = get_document(conn, doc_id) + finally: + conn.close() + item_metadata = dict(extra_meta) + if document and document["current_version_id"]: + item_metadata["supersedes_version_id"] = document["current_version_id"] if local_fixtures_dir and (local_fixtures_dir / f"{number}.pdf").exists(): fixture_file = local_fixtures_dir / f"{number}.pdf" artifact = import_manual_file( @@ -49,6 +67,7 @@ def run_seed_collection( family=family, document_type=doc_type, title=title, + extra_metadata=item_metadata, ) else: source_url = item["source_url"] @@ -59,23 +78,29 @@ def run_seed_collection( family=family, document_type=doc_type, title=title, + extra_metadata=item_metadata, ) + + is_dup = getattr(artifact, "is_duplicate", False) + if is_dup: + item_status = "unchanged" + pipe_status = "skipped" + else: + item_status = "collected" + try: + pipe_status = process_artifact_pipeline(artifact.artifact_id) + except Exception as p_err: + pipe_status = f"pipeline error: {p_err}" + collected.append(artifact) + print(f"{number} {title}: {item_status}, pipeline: {pipe_status}") + except Exception as e: - err_msg = str(e).lower() - # Skip if artifact already exists or sha256 UNIQUE constraint - if "already exists" in err_msg or "unique constraint" in err_msg: - skipped.append((number, title, "already exists")) - continue - # Log failure but continue with remaining items - skipped.append((number, title, str(e))) + err_msg = str(e) + if "already exists" in err_msg.lower() or "unique constraint" in err_msg.lower(): + print(f"{number} {title}: unchanged, pipeline: skipped") + else: + print(f"{number} {title}: failed, pipeline: skipped ({err_msg[:80]})") continue - if skipped: - import sys - - for number, title, reason in skipped: - short_reason = reason if "already exists" in reason else reason[:120] - print(f" ⚠ Skipped {number} ({title}): {short_reason}", file=sys.stderr) - return collected diff --git a/src/mesa_legal_data/harvest/discovery/resmi_gazete.py b/src/mesa_legal_data/harvest/discovery/resmi_gazete.py index eded76d..f9d3b46 100644 --- a/src/mesa_legal_data/harvest/discovery/resmi_gazete.py +++ b/src/mesa_legal_data/harvest/discovery/resmi_gazete.py @@ -9,6 +9,7 @@ from mesa_legal_data.harvest.models import DiscoveredDocument from mesa_legal_data.harvest.normalization import build_canonical_key +from mesa_legal_data.parsers.text_normalizer import normalize_text from mesa_legal_data.sources.url_fetcher import fetch_discovery_html SECTION_TYPE_MAP: dict[str, tuple[str, str]] = { @@ -23,6 +24,26 @@ "DANIŞTAY KARARLARI": ("decision", "court_decision"), } +HEADER_SECTION_MAP: dict[str, str] = { + "KANUNLAR": "KANUNLAR", + "KANUN": "KANUNLAR", + "CUMHURBAŞKANLIĞI KARARNAMELERİ": "CUMHURBAŞKANLIĞI KARARNAMELERİ", + "CUMHURBAŞKANLIĞI KARARNAMESİ": "CUMHURBAŞKANLIĞI KARARNAMELERİ", + "CUMHURBAŞKANI KARARLARI": "CUMHURBAŞKANI KARARLARI", + "CUMHURBAŞKANI KARARI": "CUMHURBAŞKANI KARARLARI", + "YÖNETMELİKLER": "YÖNETMELİKLER", + "YÖNETMELİK": "YÖNETMELİKLER", + "TEBLİĞLER": "TEBLİĞLER", + "TEBLİĞ": "TEBLİĞLER", + "YARGI BÖLÜMÜ": "YARGI BÖLÜMÜ", + "ANAYASA MAHKEMESİ KARARLARI": "ANAYASA MAHKEMESİ KARARLARI", + "ANAYASA MAHKEMESİ KARARI": "ANAYASA MAHKEMESİ KARARLARI", + "YARGITAY KARARLARI": "YARGITAY KARARLARI", + "YARGITAY KARARI": "YARGITAY KARARLARI", + "DANIŞTAY KARARLARI": "DANIŞTAY KARARLARI", + "DANIŞTAY KARARI": "DANIŞTAY KARARLARI", +} + EXCLUDED_SECTIONS = { "İLÂN BÖLÜMÜ", "ARTIRMA, EKSİLTME VE İHALE İLÂNLARI", @@ -30,6 +51,38 @@ "İLAN BÖLÜMÜ", } +ALL_EXCLUDED_HEADERS: set[str] = { + "İLÂN BÖLÜMÜ", + "İLAN BÖLÜMÜ", + "ARTIRMA, EKSİLTME VE İHALE İLÂNLARI", + "ARTIRMA, EKSİLTME VE İHALE İLÂNI", + "ARTIRMA, EKSİLTME VE İHALE İLANLARI", + "ARTIRMA, EKSİLTME VE İHALE İLANI", + "ÇEŞİTLİ İLÂNLAR", + "ÇEŞİTLİ İLÂN", + "ÇEŞİTLİ İLANLAR", + "ÇEŞİTLİ İLAN", + "İLÂNLAR", + "İLANLAR", + "İLÂN", + "İLAN", + "YARGI İLÂNLARI", + "YARGI İLÂNI", + "YARGI İLANLARI", + "YARGI İLANI", +} + +OTHER_SECTION_HEADERS: set[str] = { + "TBMM KARARI", + "TBMM KARARLARI", + "KURUL KARARI", + "KURUL KARARLARI", + "ATAMA KARARI", + "ATAMA KARARLARI", + "GENELGE", + "GENELGELER", +} + NON_DOCUMENT_STEMS = { "index", "default", @@ -110,20 +163,24 @@ def parse_html_fihrist(self, html_content: str, pub_date: date, page_url: str) - elements = soup.find_all(["h1", "h2", "h3", "h4", "div", "p", "a"]) for el in elements: - text_upper = el.get_text(strip=True).upper() - - # Section header detection - for sec_key in SECTION_TYPE_MAP: - if sec_key in text_upper: - current_section = sec_key - break - - for ex_key in EXCLUDED_SECTIONS: - if ex_key in text_upper: - current_section = ex_key - break - - if current_section in EXCLUDED_SECTIONS: + text_norm = re.sub(r"\s+", " ", el.get_text() or "").strip() + text_upper = text_norm.upper() + + # Section header detection: exact match only + if el.name != "a": + if text_upper in HEADER_SECTION_MAP: + current_section = HEADER_SECTION_MAP[text_upper] + elif text_upper in ALL_EXCLUDED_HEADERS: + current_section = text_upper + elif text_upper in OTHER_SECTION_HEADERS: + current_section = text_upper + + if ( + current_section in EXCLUDED_SECTIONS + or current_section in ALL_EXCLUDED_HEADERS + or "İLAN" in current_section + or "İLÂN" in current_section + ): continue if el.name == "a" and el.get("href"): @@ -132,10 +189,18 @@ def parse_html_fihrist(self, html_content: str, pub_date: date, page_url: str) - continue full_url = urljoin(page_url, href) + if "ilanlar" in full_url.lower(): + continue + if not is_resmi_gazete_document_link(full_url, pub_date): continue - title = el.get_text(strip=True) or "Resmî Gazete Belgesi" + raw_title = el.get_text() or "" + cleaned_title = normalize_text(raw_title) + title = " ".join(line.strip() for line in cleaned_title.splitlines() if line.strip()) + title = re.sub(r"^[\s\u2010-\u2015\u2212\-–—]+", "", title).strip() + if not title: + title = "Resmî Gazete Belgesi" if current_section in SECTION_TYPE_MAP: family, doc_type = SECTION_TYPE_MAP[current_section] @@ -148,7 +213,31 @@ def parse_html_fihrist(self, html_content: str, pub_date: date, page_url: str) - doc_id_part = link_stem canonical_key = build_canonical_key("resmi_gazete", family, doc_type, pub_date.isoformat(), doc_id_part) - document_id = f"tr:{family}:{doc_type}:rg-{doc_id_part}" + + num_id: str | None = None + if doc_type == "law": + m_law = re.search(r"^\s*(\d{3,5})\b", title) + if m_law: + num_id = m_law.group(1) + elif doc_type == "presidential_decree": + m_decree = re.search(r"Kararname\s+Numaras[ıi]\s*:\s*(\d+)", title, re.IGNORECASE) + if not m_decree: + m_decree = re.search( + r"\b(\d+)\s+Say[ıi]l[ıi]\s+Cumhurba[şs]kanl[ıi][ğg][ıi]\s+Kararnamesi", + title, + re.IGNORECASE, + ) + if m_decree: + num_id = m_decree.group(1) + elif doc_type == "presidential_decision": + m_dec = re.search(r"Karar\s+Say[ıi]s[ıi]\s*:\s*(\d+)", title, re.IGNORECASE) + if m_dec: + num_id = m_dec.group(1) + + if num_id: + document_id = f"tr:{family}:{doc_type}:{num_id}" + else: + document_id = f"tr:{family}:{doc_type}:rg-{doc_id_part}" discovered.append( DiscoveredDocument( diff --git a/src/mesa_legal_data/parsers/decision.py b/src/mesa_legal_data/parsers/decision.py index 59cdfd7..f898057 100644 --- a/src/mesa_legal_data/parsers/decision.py +++ b/src/mesa_legal_data/parsers/decision.py @@ -18,8 +18,8 @@ class ParsedDecision(BaseModel): verdict: str | None = None -ESAS_NO_PATTERN = re.compile(r"Esas\s*No\s*[:\s]*([0-9]{4}\s*/\s*[0-9]+)", re.IGNORECASE) -KARAR_NO_PATTERN = re.compile(r"Karar\s*No\s*[:\s]*([0-9]{4}\s*/\s*[0-9]+)", re.IGNORECASE) +ESAS_NO_PATTERN = re.compile(r"Esas\s*(?:No|Sayısı)\s*[:\s]*([0-9]{4}\s*/\s*[0-9]+)", re.IGNORECASE) +KARAR_NO_PATTERN = re.compile(r"Karar\s*(?:No|Sayısı)\s*[:\s]*([0-9]{4}\s*/\s*[0-9]+)", re.IGNORECASE) KARAR_TARIHI_PATTERN = re.compile(r"Karar\s*Tarihi\s*[:\s]*([0-9]{1,2}[./-][0-9]{1,2}[./-][0-9]{4})", re.IGNORECASE) diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index a6cb162..5eab8e0 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -237,8 +237,8 @@ def process_artifact_pipeline( elif fam == "decision": dec_parsed_pre = parse_decision_text(canonical_text) raw_ver_date = ( - getattr(dec_parsed_pre, "decision_date", None) - or meta_dict.get("decision_date") + meta_dict.get("decision_date") + or getattr(dec_parsed_pre, "decision_date", None) or meta_dict.get("publication_date") or meta_dict.get("snapshot_date") ) @@ -451,6 +451,13 @@ def process_artifact_pipeline( else: # Decision family dec_parsed = parse_decision_text(canonical_text) + dec_parsed = dec_parsed.model_copy( + update={ + field: meta_dict[field] + for field in ("court", "chamber", "esas_no", "karar_no", "decision_date") + if meta_dict.get(field) + } + ) dec_id = build_decision_id( dec_parsed.court or "unknown", dec_parsed.chamber, diff --git a/src/mesa_legal_data/publisher/client.py b/src/mesa_legal_data/publisher/client.py index 5d5dc77..04f8b6d 100644 --- a/src/mesa_legal_data/publisher/client.py +++ b/src/mesa_legal_data/publisher/client.py @@ -1,4 +1,6 @@ +import json import os +import re import time from typing import Any from urllib.parse import urlparse @@ -14,6 +16,18 @@ SourceChunk, ) +# Mirrors mesa_storage/legal_identity.py::LAW_IDS in MESA, without 2709: MESA main +# rejects "2709" at insert and dead-letters "Anayasa" (its canonical form fails its own +# re-check), so Constitution chunks are sent without legal_identity. +MESA_LEGAL_IDENTITY_LAW_NUMBERS = frozenset({"4721", "6098", "6100", "6102", "2577", "5237", "5271", "4857", "6698"}) + +# MESA input_validation.MAX_METADATA_BYTES, measured the same way MESA measures it. +MESA_METADATA_MAX_BYTES = 16 * 1024 + + +def _metadata_size(metadata: dict[str, Any]) -> int: + return len(json.dumps(metadata, sort_keys=True, default=str).encode()) + class MesaClientError(Exception): """Base error for MESA client interactions.""" @@ -404,10 +418,17 @@ def build_memory_insert_payload( "mesa_data_char_end": chunk.char_end, "mesa_data_content_hash": chunk.content_hash, } + # MESA rejects metadata over 16 KB; drop trailing citations until it fits. + citations = list(metadata.get("citations") or []) + while citations and _metadata_size(metadata) > MESA_METADATA_MAX_BYTES: + citations.pop() + metadata["citations"] = citations + if "citations" in metadata and not citations: + del metadata["citations"] source_ref = metadata.get("authoritative_source_ref") or metadata.get("source_url") if not isinstance(source_ref, str) or not source_ref.strip(): source_ref = f"mesa-data://releases/{metadata.get('release_id', 'current')}/documents/{chunk.document_id}/revisions/{chunk.version_id}/chunks/{chunk.chunk_id}" - return { + payload = { "session_id": session_id, "dataset_id": self.settings.dataset_id, "document_id": chunk.document_id, @@ -424,6 +445,16 @@ def build_memory_insert_payload( "metadata": metadata, "idempotency_key": idempotency_key, } + law_number = str(metadata.get("law_number", "")) + article_number = str(metadata.get("article_number", "")) + if ( + chunk.chunk_type == "article" + and metadata.get("article_kind") == "standard" + and law_number in MESA_LEGAL_IDENTITY_LAW_NUMBERS + and re.fullmatch(r"\d+[A-Za-zÇĞİÖŞÜçğıöşü]?", article_number) + ): + payload["legal_identity"] = {"law_id": law_number, "article_id": article_number} + return payload def publish_source_chunk( self, diff --git a/src/mesa_legal_data/publisher/engine.py b/src/mesa_legal_data/publisher/engine.py index 65f8db2..15038e0 100644 --- a/src/mesa_legal_data/publisher/engine.py +++ b/src/mesa_legal_data/publisher/engine.py @@ -336,11 +336,11 @@ def build_delivery_plan( ) # Canonical citation edges are release-owned records, not text inferred - # by the publisher. Carry only citations whose authoritative source span - # overlaps this chunk. Target versions stay null until a corpus-aware or - # human resolver verifies the exact legal version. + # by the publisher. Carry the targets of citations whose authoritative + # source span overlaps this chunk. MESA metadata allows no objects inside + # lists, so each citation is its target id; unresolved ones are dropped. for chunk in chunks: - chunk_citations: list[dict[str, Any]] = [] + chunk_citations: list[str] = [] for citation in citations: span = citation.get("source_span") or {} start = span.get("char_start") @@ -349,21 +349,9 @@ def build_delivery_plan( continue if start >= chunk.char_end or end <= chunk.char_start: continue - chunk_citations.append( - { - "citation_id": citation.get("id"), - "source_record_id": citation.get("source_record_id"), - "source_version_id": citation.get("source_version_id") or v_id, - "target_legislation_id": citation.get("target_legislation_id"), - "target_article_id": citation.get("target_article_id"), - "target_version_id": citation.get("target_version_id"), - "citation_status": citation.get("citation_status"), - "relation_hint": citation.get("relation_hint"), - "char_start": start, - "char_end": end, - "fully_contained": start >= chunk.char_start and end <= chunk.char_end, - } - ) + target = citation.get("target_article_id") or citation.get("target_legislation_id") + if isinstance(target, str) and target and target not in chunk_citations: + chunk_citations.append(target) if chunk_citations: chunk.metadata["citations"] = chunk_citations diff --git a/src/mesa_legal_data/sources/bedesten.py b/src/mesa_legal_data/sources/bedesten.py new file mode 100644 index 0000000..87f7c20 --- /dev/null +++ b/src/mesa_legal_data/sources/bedesten.py @@ -0,0 +1,196 @@ +"""Collect the scoped Yargıtay boards through Bedesten's JSON endpoints.""" + +import base64 +import binascii +import json +import tempfile +from datetime import date, datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +import httpx + +from mesa_legal_data.config import load_settings +from mesa_legal_data.ids import build_decision_id +from mesa_legal_data.models import FetchedArtifact +from mesa_legal_data.pipeline import process_artifact_pipeline +from mesa_legal_data.sources import request_control +from mesa_legal_data.sources.manual import import_manual_file +from mesa_legal_data.sources.url_fetcher import ( + build_ssl_context, + detect_html_error_page, + validate_source_request, + validate_url_host, +) + +BASE_URL = "https://bedesten.adalet.gov.tr/emsal-karar/" +UNITS = { + "HGK": "Hukuk Genel Kurulu", + "CGK": "Ceza Genel Kurulu", + "IBK": "İçtihatları Birleştirme Kurulu", +} + + +def _post(endpoint: str, data: dict, *, paging: bool = False) -> dict: + url = BASE_URL + endpoint + policy = validate_source_request(source_id="yargitay", document_family="decision", url=url) + validate_url_host(url) + contact = load_settings().operator_contact.strip() + if not contact or contact.lower() in { + "test@example.com", + "example.com", + "operator@example.com", + "placeholder", + "admin@example.com", + "foo@bar.com", + }: + raise ValueError("OPERATOR_CONTACT_MISSING: configure a valid operator_contact for Bedesten") + user_agent = policy.user_agent + if contact not in user_agent: + user_agent += f" (+{contact})" + payload: dict = {"data": data, "applicationName": "UyapMevzuat"} + if paging: + payload["paging"] = True + state = request_control.get_source_request_state("yargitay", policy.concurrency) + budget = request_control.get_run_budget("yargitay", policy.max_requests_per_run) + # JSON contains base64, whose size is at most 4/3 of the decoded file, plus metadata. + max_response = 4 * ((policy.max_download_bytes + 2) // 3) + 1024 * 1024 + with ( + state.semaphore, + httpx.Client( + verify=build_ssl_context(), + follow_redirects=False, + timeout=policy.timeout_seconds, + headers={ + "User-Agent": user_agent, + "Accept": "application/json", + "AdaletApplicationName": "UyapMevzuat", + "Origin": "https://mevzuat.adalet.gov.tr", + "Referer": "https://mevzuat.adalet.gov.tr/", + }, + ) as client, + ): + for attempt in range(2): + request_control.enforce_min_interval(state, max(6, policy.min_interval_seconds)) + budget.consume() + with client.stream("POST", url, json=payload) as response: + if response.status_code == 429: + if attempt: + raise ValueError("Bedesten HTTP 429 after one retry; collection stopped") + retry_after = response.headers.get("retry-after", "") + delay = min(float(retry_after), 60) if retry_after.isdigit() else 6 + else: + if response.status_code != 200: + raise ValueError(f"Bedesten HTTP {response.status_code}; collection stopped") + if response.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ValueError("Bedesten response is not application/json") + body = bytearray() + for chunk in response.iter_bytes(): + body.extend(chunk) + if len(body) > max_response: + raise ValueError("Bedesten response exceeds source download size limit") + result = json.loads(body) + if not isinstance(result, dict) or not isinstance(result.get("data"), dict): + raise ValueError( + f"Bedesten API error: {result.get('metadata') if isinstance(result, dict) else result}" + ) + return result["data"] + request_control.sleep_func(delay) + raise ValueError("Bedesten request failed") + + +def collect_bedesten( + unit: str, + from_date: date, + to_date: date | None = None, + *, + limit: int = 10, +) -> list[FetchedArtifact]: + if unit not in UNITS: + raise ValueError("Unit must be HGK, CGK or IBK") + end = to_date or datetime.now(ZoneInfo("Europe/Istanbul")).date() + if limit < 1 or from_date < date(2021, 1, 1) or end < from_date: + raise ValueError("Use a positive limit and a date range starting on or after 2021-01-01") + request_control.reset_run_budget("yargitay") + artifacts: list[FetchedArtifact] = [] + page = 1 + page_size = min(10, limit) + while len(artifacts) < limit: + data = _post( + "searchDocuments", + { + "pageSize": page_size, + "pageNumber": page, + "itemTypeList": ["YARGITAYKARARI"], + # Empty phrases are rejected. The unit name occurs in decision headings. + "phrase": UNITS[unit], + "birimAdi": UNITS[unit], + "kararTarihiStart": from_date.isoformat() + "T00:00:00", + "kararTarihiEnd": end.isoformat() + "T23:59:59", + "sortFields": ["KARAR_TARIHI"], + "sortDirection": "desc", + }, + paging=True, + ) + items = data.get("emsalKararList", []) + if not items: + break + for item in items[: limit - len(artifacts)]: + if item["birimAdi"] != UNITS[unit]: + raise ValueError("Bedesten returned a decision outside the requested unit") + # Bedesten timestamps are UTC; the displayed date is the Turkish legal date. + if item.get("kararTarihiStr"): + decision_date = datetime.strptime(item["kararTarihiStr"], "%d.%m.%Y").date() + else: + decision_date = ( + datetime.fromisoformat(item["kararTarihi"].replace("Z", "+00:00")) + .astimezone(ZoneInfo("Europe/Istanbul")) + .date() + ) + if not from_date <= decision_date <= end: + raise ValueError("Bedesten returned a decision outside the requested dates") + doc_id = build_decision_id("YARGITAY", item["birimAdi"], item["esasNo"], item["kararNo"], "") + metadata = { + "court": "YARGITAY", + "chamber": item["birimAdi"], + "esas_no": item["esasNo"], + "karar_no": item["kararNo"], + "decision_date": decision_date.isoformat(), + "bedesten_document_id": item["documentId"], + } + content = _post("getDocumentContent", {"documentId": item["documentId"]}) + policy = validate_source_request( + source_id="yargitay", + document_family="decision", + url=BASE_URL + "getDocumentContent", + ) + mime = content.get("mimeType", "").split(";")[0].strip().lower() + if mime not in policy.allowed_content_types or mime not in {"text/html", "application/pdf"}: + raise ValueError(f"Bedesten document MIME is not allowed: {mime}") + try: + raw = base64.b64decode(content["content"], validate=True) + except (binascii.Error, KeyError, TypeError) as exc: + raise ValueError("Bedesten returned invalid base64 document content") from exc + if not raw or len(raw) > policy.max_download_bytes: + raise ValueError("Bedesten document is empty or exceeds source download size limit") + if mime == "text/html" and (error := detect_html_error_page(raw.decode("utf-8", errors="replace"))): + raise ValueError(error) + with tempfile.TemporaryDirectory(prefix="mesa-bedesten-") as temp: + file = Path(temp) / ("decision.pdf" if mime == "application/pdf" else "decision.html") + file.write_bytes(raw) + artifact = import_manual_file( + file_path=file, + source_id="yargitay", + family="decision", + document_type="yargitay_decision", + document_id=doc_id, + title=f"{item['birimAdi']} {item['esasNo']} E., {item['kararNo']} K.", + extra_metadata=metadata, + ) + status = "skipped" if artifact.is_duplicate else process_artifact_pipeline(artifact.artifact_id) + print(f"{doc_id}: {'unchanged' if artifact.is_duplicate else 'collected'}, pipeline: {status}") + artifacts.append(artifact) + if page * page_size >= data.get("total", page * page_size + 1): + break + page += 1 + return artifacts diff --git a/src/mesa_legal_data/sources/manual.py b/src/mesa_legal_data/sources/manual.py index 7d9a417..947c092 100644 --- a/src/mesa_legal_data/sources/manual.py +++ b/src/mesa_legal_data/sources/manual.py @@ -36,6 +36,7 @@ def import_manual_file( publication_date: str | None = None, stable_key: str | None = None, sources_yaml_path: Path | None = None, + extra_metadata: dict | None = None, ) -> FetchedArtifact: if not file_path.exists(): raise FileNotFoundError(f"File not found: {file_path}") @@ -61,7 +62,72 @@ def import_manual_file( with open(file_path, "rb") as f: artifact_sha256 = hash_stream(f) - # 3. Path construction + # 3. Path construction & Duplicate check + artifact_id = f"sha256:{artifact_sha256}" + conn = get_connection() + existing = get_artifact(conn, artifact_id) + if existing: + if existing.get("document_id") and existing["document_id"] != document_id: + conn.close() + raise ArtifactDocumentCollisionError( + "ARTIFACT_DOCUMENT_COLLISION: identical payload is already bound to a different document" + ) + doc_key = stable_key if stable_key else secure_slug(document_id) + upsert_document( + conn=conn, + document_id=document_id, + family=family, + document_type=document_type, + jurisdiction=jurisdiction, + title=title, + stable_key=doc_key, + lifecycle_status="fetched", + ) + dup_meta = {} + if extra_metadata: + dup_meta.update(extra_metadata) + dup_meta["publication_date"] = publication_date + insert_artifact( + conn=conn, + artifact_id=existing["artifact_id"], + document_id=document_id, + source_id=existing["source_id"], + source_url=existing["source_url"], + retrieved_at=existing["retrieved_at"], + fetch_method=existing["fetch_method"], + http_status=existing["http_status"], + declared_content_type=existing["declared_content_type"], + detected_content_type=existing["detected_content_type"], + byte_size=existing["byte_size"], + sha256=artifact_sha256, + raw_path=existing["raw_path"], + etag=existing["etag"], + last_modified=existing["last_modified"], + transport_status=existing["transport_status"], + error_code=existing["error_code"], + metadata_json=json.dumps(dup_meta), + ) + refreshed = get_artifact(conn, artifact_id) or existing + conn.close() + return FetchedArtifact( + artifact_id=artifact_id, + document_id=document_id or existing["document_id"], + source_id=source_id, + source_url=f"file://{file_path.resolve()}", + retrieved_at=existing["retrieved_at"], + fetch_method="manual", + http_status=200, + declared_content_type=detected_mime, + detected_content_type=detected_mime, + byte_size=existing["byte_size"], + sha256=artifact_sha256, + raw_path=existing["raw_path"], + transport_status=existing.get("transport_status", "verified"), + is_duplicate=True, + metadata=json.loads(refreshed.get("metadata_json") or "{}"), + ) + conn.close() + ext = file_path.suffix if file_path.suffix else ".bin" doc_key = stable_key if stable_key else secure_slug(document_id) year = datetime.now(UTC).year @@ -83,26 +149,30 @@ def import_manual_file( # 5. Metadata json retrieved_at = datetime.now(UTC).isoformat() byte_size = os.path.getsize(file_path) - artifact_id = f"sha256:{artifact_sha256}" - meta_dict = { - "artifact_id": artifact_id, - "document_key": doc_key, - "source_id": source_id, - "source_url": f"file://{file_path.resolve()}", - "retrieved_at": retrieved_at, - "publication_date": publication_date, - "fetch_method": "manual", - "http_status": 200, - "declared_content_type": detected_mime, - "detected_content_type": detected_mime, - "byte_size": byte_size, - "sha256": artifact_sha256, - "etag": None, - "last_modified": None, - "collector_version": "1.0.0", - "access_policy_version": policy.policy_version, - } + meta_dict = {} + if extra_metadata: + meta_dict.update(extra_metadata) + meta_dict.update( + { + "artifact_id": artifact_id, + "document_key": doc_key, + "source_id": source_id, + "source_url": f"file://{file_path.resolve()}", + "retrieved_at": retrieved_at, + "publication_date": publication_date, + "fetch_method": "manual", + "http_status": 200, + "declared_content_type": detected_mime, + "detected_content_type": detected_mime, + "byte_size": byte_size, + "sha256": artifact_sha256, + "etag": None, + "last_modified": None, + "collector_version": "1.0.0", + "access_policy_version": policy.policy_version, + } + ) metadata_path = full_target_payload.parent / "metadata.json" import io @@ -194,6 +264,7 @@ def import_manual_url( publication_date: str | None = None, stable_key: str | None = None, sources_yaml_path: Path | None = None, + extra_metadata: dict | None = None, ) -> FetchedArtifact: import io @@ -262,6 +333,11 @@ def import_manual_url( stable_key=doc_key, lifecycle_status="fetched", ) + dup_meta = {} + if extra_metadata: + dup_meta.update(extra_metadata) + dup_meta["publication_date"] = publication_date + insert_artifact( conn=conn, artifact_id=existing["artifact_id"], @@ -280,7 +356,7 @@ def import_manual_url( last_modified=existing["last_modified"], transport_status=existing["transport_status"], error_code=existing["error_code"], - metadata_json=json.dumps({"publication_date": publication_date}), + metadata_json=json.dumps(dup_meta), ) refreshed = get_artifact(conn, artifact_id) or existing conn.close() @@ -324,24 +400,29 @@ def import_manual_url( retrieved_at = datetime.now(UTC).isoformat() byte_size = os.path.getsize(temp_file) - meta_dict = { - "artifact_id": artifact_id, - "document_key": doc_key, - "source_id": source_id, - "source_url": url, - "retrieved_at": retrieved_at, - "fetch_method": "manual_url", - "http_status": status_code, - "declared_content_type": declared_content_type, - "detected_content_type": detected_mime, - "byte_size": byte_size, - "sha256": artifact_sha256, - "etag": headers.get("etag"), - "last_modified": headers.get("last-modified"), - "publication_date": publication_date, - "collector_version": "1.0.0", - "access_policy_version": policy.policy_version, - } + meta_dict = {} + if extra_metadata: + meta_dict.update(extra_metadata) + meta_dict.update( + { + "artifact_id": artifact_id, + "document_key": doc_key, + "source_id": source_id, + "source_url": url, + "retrieved_at": retrieved_at, + "fetch_method": "manual_url", + "http_status": status_code, + "declared_content_type": declared_content_type, + "detected_content_type": detected_mime, + "byte_size": byte_size, + "sha256": artifact_sha256, + "etag": headers.get("etag"), + "last_modified": headers.get("last-modified"), + "publication_date": publication_date, + "collector_version": "1.0.0", + "access_policy_version": policy.policy_version, + } + ) metadata_path = full_target_payload.parent / "metadata.json" meta_bytes = json.dumps(meta_dict, indent=2).encode("utf-8") diff --git a/tests/acceptance/test_security_and_scale_gate.py b/tests/acceptance/test_security_and_scale_gate.py index 50785da..8e50222 100644 --- a/tests/acceptance/test_security_and_scale_gate.py +++ b/tests/acceptance/test_security_and_scale_gate.py @@ -52,12 +52,30 @@ def test_master_security_and_scale_acceptance_gate(tmp_path, monkeypatch): ) # 2. Disabled Yargitay source -> FAILS + sources_yaml_path = tmp_path / "sources.yaml" + sources_yaml_path.write_text( + 'version: "1.0.0"\n' + "sources:\n" + " yargitay:\n" + " name: Disabled Yargitay\n" + " authority: Yargitay\n" + " base_url: https://karararama.yargitay.gov.tr/\n" + " access_mode: manual\n" + " enabled: false\n" + " families: [decision]\n" + " source_role: official_case_law\n" + ' policy_version: "1.0.0"\n' + " http: {user_agent: MESA-Legal-Data-Test/1.0}\n" + " allowed_content_types: [text/html]\n", + encoding="utf-8", + ) with pytest.raises(SourcePolicyError, match="SOURCE_DISABLED"): import_manual_url( url="https://karararama.yargitay.gov.tr/doc.html", source_id="yargitay", document_id="tr:case-law:yargitay:2026:1", family="decision", + sources_yaml_path=sources_yaml_path, ) # 3. Non-HTTPS (HTTP) URL -> FAILS diff --git a/tests/conftest.py b/tests/conftest.py index 4c946f7..87f8d60 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import ipaddress +import os import socket import time @@ -6,6 +7,26 @@ from mesa_legal_data.sources import request_control + +@pytest.fixture(scope="session", autouse=True) +def isolate_session_data_root(tmp_path_factory): + session_tmp = tmp_path_factory.mktemp("mesa_data_session") + staging_db = session_tmp / "mesa_staging.sqlite" + old_data_root = os.environ.get("MESA_DATA_DATA_ROOT") + old_staging_db = os.environ.get("MESA_DATA_MESA_STAGING_DB") + os.environ["MESA_DATA_DATA_ROOT"] = str(session_tmp) + os.environ["MESA_DATA_MESA_STAGING_DB"] = str(staging_db) + yield session_tmp + if old_data_root is not None: + os.environ["MESA_DATA_DATA_ROOT"] = old_data_root + else: + os.environ.pop("MESA_DATA_DATA_ROOT", None) + if old_staging_db is not None: + os.environ["MESA_DATA_MESA_STAGING_DB"] = old_staging_db + else: + os.environ.pop("MESA_DATA_MESA_STAGING_DB", None) + + KNOWN_TEST_HOSTS = { "mevzuat.gov.tr": ["1.1.1.1"], "www.mevzuat.gov.tr": ["1.1.1.1"], diff --git a/tests/golden/bedesten/document-1231234400.json b/tests/golden/bedesten/document-1231234400.json new file mode 100644 index 0000000..f454871 --- /dev/null +++ b/tests/golden/bedesten/document-1231234400.json @@ -0,0 +1 @@ +{"data":{"content":"PGh0bWw+PGhlYWQ+PG1ldGEgaHR0cC1lcXVpdj0iQ29udGVudC1UeXBlIiBjb250ZW50PSJ0ZXh0L2h0bWw7IGNoYXJzZXQ9VVRGLTgiPjwvaGVhZD4gPGJvZHkgbGVmdG1hcmdpbj0iMjUiIHRvcG1hcmdpbj0iMjAiIGZvbnQgZmFjZT0iVmVyZGFuYSIgc2l6ZT0iMiI+PGI+PGZvbnQgZmFjZT0iVmVyZGFuYSIgc2l6ZT0iMiI+SHVrdWsgR2VuZWwgS3VydWx1IMKgwqDCoMKgwqDCoMKgwqAyMDI1LzUxMCBFLiDCoCwgwqAyMDI2LzUyOSBLLjwvZm9udD48L2I+PHVsPjxsaSBzdHlsZT0iZm9udC1mYW1pbHk6VmVyZGFuYTtmb250LXNpemU6MTI7Zm9udC13ZWlnaHQ6Ym9sZCI+PC9saT48L3VsPjx1bCBzdHlsZT0ibGlzdC1zdHlsZS10eXBlOiBjaXJjbGU7Zm9udC1mYW1pbHk6VmVyZGFuYTtjb2xvcjojMTA0ZDk2O2ZvbnQtc2l6ZToxMiI+PC91bD48YnI+CjxiPjxmb250IGZhY2U9IlZlcmRhbmEiIHNpemU9IjIiPiLEsMOndGloYXQgTWV0bmkiPC9mb250PjwvYj48cCBhbGlnbj0ianVzdGlmeSI+PGZvbnQgZmFjZT0iVmVyZGFuYSIgc2l6ZT0iMiI+TUFIS0VNRVPEsAk6S2FkYXN0cm8gTWFoa2VtZXNpPGJyPlNBWUlTSQk6IDIwMjQvNjggRS4sIDIwMjUvMTQgSy48YnI+w5ZaRUwgREHEsFJFIEtBUkFSSQk6IFlhcmfEsXRheSAoS2FwYXTEsWxhbikgMTYuIEh1a3VrIERhaXJlc2luaW4gMzEuMDMuMjAyMSB0YXJpaGxpIHZlIDxicj4JICAyMDE5LzExMSBFc2FzLCAyMDIxLzMwMDcgS2FyYXIgc2F5xLFsxLEgQk9aTUEga2FyYXLEsTxicj48YnI+VGFyYWZsYXIgYXJhc8SxbmRha2kga2FkYXN0cm8gdGVzcGl0aW5lIGl0aXJheiBkYXZhc8SxbmRhbiBkb2xhecSxIHlhcMSxbGFuIHlhcmfEsWxhbWEgc29udW5kYSBNYWhrZW1lY2UgYXPEsWwgZGF2YW7EsW4gcmVkZGluZSwgYmlybGXFn2VuIGRhdmFuxLFuIGthYnVsw7xuZSBrYXJhciB2ZXJpbG1pxZ90aXIuPGJyPk1haGtlbWVjZSB2ZXJpbGVuIGthcmFyxLFuIGRhdmFjxLEgLi4uIGlsZSBkYXZhbMSxIC4uLiB2ZWtpbGkgdGFyYWbEsW5kYW4gdGVteWl6IGVkaWxtZXNpIMO8emVyaW5lIFlhcmfEsXRheSAoS2FwYXTEsWxhbikgMTYuIEh1a3VrIERhaXJlc2luY2UgeWFwxLFsYW4gaW5jZWxlbWUgc29udW5kYSBib3p1bG11xZ8sICBNYWhrZW1lIHRhcmFmxLFuZGFuIMOWemVsIERhaXJlIGJvem1hIGthcmFyxLFuYSBrYXLFn8SxIGRpcmVuaWxtacWfdGlyLjxicj5EaXJlbm1lIGthcmFyxLEgZGF2YWzEsSAuLi4gdmVraWxpIHRhcmFmxLFuZGFuIHRlbXlpeiBlZGlsbWVrbGU7IGtlc2lubGlrLCBzw7xyZSwgdGVteWl6IMWfYXJ0xLEgdmUgZGnEn2VyIHVzdWwgZWtzaWtsaWtsZXJpIHnDtm7DvG5kZW4geWFwxLFsYW4gw7ZuIGluY2VsZW1lIHNvbnVjdW5kYSwgdGVteWl6IGRpbGVrw6dlc2luaW4ga2FidWzDvG5lIGthcmFyIHZlcmlsZGlrdGVuIHNvbnJhIFRldGtpayBIw6JraW1pIHRhcmFmxLFuZGFuIGhhesSxcmxhbmFuIGfDvG5kZW0gdmUgZG9zeWFkYWtpIGJlbGdlbGVyIGluY2VsZW5pcCBnZXJlxJ9pIGTDvMWfw7xuw7xsZMO8Ojxicj5BLiDDlm4gU29ydW48YnI+SHVrdWsgR2VuZWwgS3VydWx1bmRhIHlhcMSxbGFuIGfDtnLDvMWfbWVsZXIgc8SxcmFzxLFuZGEgacWfaW4gZXNhc8SxbsSxbiBpbmNlbGVubWVzaW5kZW4gw7ZuY2UsIGRpcmVubWUgb2xhcmFrIGFkbGFuZMSxcsSxbGFuIGthcmFyxLFuIHllbmkgZGVsaWwgdmUgIGdlcmVrw6dleWUgZGF5YWzEsSB5ZW5pIGjDvGvDvG0gbml0ZWxpxJ9pbmRlIG9sdXAgb2xtYWTEscSfxLEsIGJ1cmFkYW4gdmFyxLFsYWNhayBzb251Y2EgZ8O2cmUgdGVteWl6IGluY2VsZW1lc2luaW4gSHVrdWsgR2VuZWwgS3VydWx1bmNhIG3EsSB5b2tzYSDDlnplbCBEYWlyZWNlIG1pIHlhcMSxbG1hc8SxIGdlcmVrdGnEn2kgaHVzdXN1IMO2biBzb3J1biBvbGFyYWsgdGFydMSxxZ/EsWzEsXAgZGXEn2VybGVuZGlyaWxtacWfdGlyLjxicj5CLiBHZXJla8OnZTxicj4xLiBCaWxpbmRpxJ9pIMO8emVyZSBkaXJlbm1lIGthcmFyxLFuxLFuIHZhcmzEscSfxLFuZGFuIHPDtnogZWRpbGViaWxtZXNpIGnDp2luIG1haGtlbWUgYm96bWEga2FyYXLEsW5kYW4gZXNpbmxlbmVyZWsgeWVuaSBiaXIgZGVsaWwgdG9wbGFtYWRhbiDDtm5jZWtpIGRlbGlsbGVyIMOnZXLDp2V2ZXNpbmRlIGthcmFyIHZlcm1lbGk7IGdlcmVrw6dlc2luaSDDtm5jZWtpIGthcmFyxLFuYSBnw7ZyZSBnZW5pxZ9sZXRlYmlsaXJzZSBkZSBkZcSfacWfdGlybWVtZWxpZGlyLjxicj4yLiBCYcWfa2EgYmlyIGFubGF0xLFtbGEsIG1haGtlbWVuaW4geWVuaSBiaXIgZGVsaWxlIGRheWFubWFrIHZleWEgYm96bWFkYW4gZXNpbmxlbm1layBzdXJldGl5bGUgZ2VyZWvDp2VzaW5pIGRlxJ9pxZ90aXJlcmVrIHZleWEgZGFoYSDDtm5jZSDDvHplcmluZGUgZHVybWFkxLHEn8SxIGJpciBodXN1c3UgYm96bWFkYSBpxZ9hcmV0IG9sdW5hbiDFn2VraWxkZSBkZcSfZXJsZW5kaXJlcmVrIGthcmFyIHZlcm1pxZ8gb2xtYXPEsSBow6JsaW5kZSwgZGlyZW5tZSBrYXJhcsSxbsSxbiB2YXJsxLHEn8SxbmRhbiBzw7Z6IGVkaWxlbWV6Ljxicj4zLiBZYXJnxLF0YXnEsW4gaXN0aWtyYXIga2F6YW5txLHFnyBpw6d0aWhhdGxhcsSxbmEgZ8O2cmUsIG1haGtlbWVjZSBkaXJlbm1lIGthcmFyxLEgdmVyaWxzZSBkYWhpIGJvem1hIGthcmFyxLFuZGEgdGFydMSxxZ/EsWxtYXPEsSBnZXJla2VuIGh1c3VzbGFyxLEgdGFydMSxxZ9tYWssIGJvem1hIHNvbnJhc8SxIHlhcMSxbGFuIGFyYcWfdMSxcm1hLCBpbmNlbGVtZSB2ZXlhIHRvcGxhbmFuIHllbmkgZGVsaWxsZXJlIGRheWFubWFrLCDDtm5jZWtpIGthcmFyZGEgeWVyIGFsbWF5YW4gdmUgw5Z6ZWwgRGFpcmUgZGVuZXRpbWluZGVuIGdlw6dtZW1pxZ8gb2xhbiB5ZW5pIHZlIGRlxJ9pxZ9payBnZXJla8OnZSBpbGUgaMO8a8O8bSBrdXJtYWsgc3VyZXRpeWxlIHZlcmlsZW4ga2FyYXIgZGlyZW5tZSBrYXJhcsSxIG9sbWF5xLFwIHllbmkgaMO8a8O8bSBvbGFyYWsga2FidWwgZWRpbG1la3RlZGlyLjxicj40LiBTb211dCBvbGF5ZGE7IE1haGtlbWVjZSwgw5Z6ZWwgRGFpcmVuaW4gYmlyaW5jaSBib3ptYSBrYXJhcsSxbmEgdXl1bGFyYWsgeWFwxLFsYW4geWFyZ8SxbGFtYSBzb251Y3VuZGEgS2FkYXN0cm8gS2FudW514oCZbnVuIDE0LiBtYWRkZXNpbmRlIHRhcHVkYSBrYXnEsXRsxLEgb2xtYXlhbiB0YcWfxLFubWF6YSBhcmFsxLFrc8SxeiB2ZSDDp2VracWfbWVzaXogZW4gYXogeWlybWkgecSxbGRhbiBiZXJpIG1hbGlrICBzxLFmYXTEsXlsYSB6aWx5ZXQgb2xhbiBracWfaSAgYWTEsW5hIHRlc2NpbCBlZGlsbWVzaW5lIGlta8OibiBzYcSfbGFuZMSxxJ/EsSwgYnVuYSBnw7ZyZSBkYXZhY8SxIHlhcmFyxLFuYSBrYXphbmTEsXLEsWPEsSB6YW1hbmHFn8SxbcSxIGlsZSBpa3Rpc2FwIGtvxZ91bGxhcsSxbsSxbiBnZXLDp2VrbGXFn3RpxJ9pIGdlcmVrw6dlc2l5bGUgZGF2YWPEsSAuLi4gQmVsZWRpeWUgQmHFn2thbmzEscSfxLFuxLFuIGRhdmFzxLFuxLFuIHJlZGRpbmUsIGRhdmFjxLEgLi4u4oCZbsSxbiBkYXZhc8SxbsSxbiAga2FidWzDvG5lIGthcmFyIHZlcmlsbWnFn3Rpci4gPGJyPjUuIEthcmFyxLFuIGRhdmFjxLEgLi4uIHZla2lsaSBpbGUgZGF2YWzEsSAuLi4gdGVtc2lsY2lzaSB0YXJhZsSxbmRhbiB0ZW15aXppIMO8emVyaW5lIMOWemVsIERhaXJlY2UgbWFoa2VtZWNlIGJvem1hIGlsw6JtxLFuYSB1eXVsZHXEn3UgaMOibGRlLCBib3ptYSBpbMOibcSxIGdlcmVrbGVyaW5pbiB0YW0gb2xhcmFrIHllcmluZSBnZXRpcmlsbWVkacSfaSwgYm96bWEgaWzDom3EsW5hIHV5dWxtYWtsYSB0YXJhZmxhciBsZWhpbmUgdXN1bGkga2F6YW7EsWxtxLHFnyBoYWsgb2x1xZ9hY2HEn8SxLCBiaWxpcmtpxZ9pIHJhcG9ydW5kYSB0YcWfxLFubWF6IMO8emVyaW5kZSBtZXl2ZSBhxJ9hw6dsYXLEsW7EsW4gYnVsdW5kdcSfdSBiZWxpcnRpbG1pxZ8gaXNlIGRlIHppcmFhdMOnaSBiaWxpcmtpxZ9pIHJhcG9ydW5kYSBidSBodXN1c3RhIGhlcmhhbmdpIGJpciBhw6fEsWtsYW1heWEgeWVyIHZlcmlsbWVkacSfaSB2ZSBiw7Z5bGVsaWtsZSByYXBvcmxhciBhcmFzxLFuZGEgdGHFn8Sxbm1heiDDvHplcmluZGUgbWV5dmUgYcSfYWPEsSBidWx1bnVwIGJ1bHVubWFkxLHEn8SxIGh1c3VzdW5kYSBhw6fEsWsgw6dlbGnFn2tpIG9sdcWfdHXEn3UsIHRhxZ/EsW5tYXrEsW4gbml0ZWxpxJ9pIHZlIMO8emVyaW5kZSBzw7xyZMO8csO8bGVuIHppbHlldGxpxJ9pbmkgxZ9la2xpIHZlIHPDvHJlc2luaW4gYmVsaXJsZW5tZXNpIGHDp8Sxc8SxbmRhbiBidSBodXN1c3RhIGVuIMO2bmVtbGkgZGVsaWwgb2xhbiBoYXZhIGZvdG/En3JhZmxhcsSxbmRhbiBkYSB5YXJhcmxhbsSxbG1hZMSxxJ/EsSwgYnUgxZ9la2lsZGUgZWtzaWsgIGFyYcWfdMSxcm1hIHZlIGluY2VsZW1leWUgdmUga2VuZGkgacOnaW5kZSDDp2VsacWfa2lsaSByYXBvcmEgZGF5YWzEsSBvbGFyYWsgaMO8a8O8bSBrdXJ1bGFtYXlhY2HEn8SxIGdlcmVrw6dlc2l5bGUgaMO8a23DvG4gYm96dWxtYXPEsW5hIGthcmFyIHZlcmlsbWnFn3Rpci48YnI+Ni4gw5Z6ZWwgRGFpcmVuaW4gYm96bWEga2FyYXLEsSDDvHplcmluZSBNYWhrZW1lY2UsIGJvem1hIGthcmFyxLFuZGFuIHNvbnJhIGRhdmEga29udXN1IHRhxZ/EsW5tYXrEsW4ga29udW11bnVuIGhhdmEgZm90b8SfcmFmbGFyxLEgw7x6ZXJpbmRlIGplb2RlemkgdmUgZm90b2dyYW1ldHJpIGJpbGlya2nFn2lzaW5kZW47IGthZGFzdHJvIHBhZnRhc8SxbmRha2kga29udW11bnVuIGJpbGdpc2F5YXIgcHJvZ3JhbcSxIGFyYWPEsWzEscSfxLF5bGEgdXlkdSB2ZSBoYXZhIGZvdG/En3JhZmxhcsSxbsSxbiBha3RhcsSxbG1hc8SxIHN1cmV0aXlsZSB1eWR1IHZlIGhhdmEgZm90b8SfcmFmbGFyxLFuxLFuIGFpdCBvbGR1a2xhcsSxIHnEsWxsYXJhIGfDtnJlIHRhxZ/EsW5tYXrEsW4gbml0ZWxpxJ9pbmksIGltYXIgaWh5YXlhIGtvbnUgb2x1cCBvbG1hZMSxxJ/EsW7EsSwgb2xtdcWfIGlzZSBpbWFyIGloeWF5YSBlbiB5YWvEsW4gbmUgemFtYW4gYmHFn2xhbsSxbGTEscSfxLFuxLEgdmUgaW1hciBpaHlhbsSxbiBoYW5naSB0YXJpaHRlIHRhbWFtbGFuZMSxxJ/EsW7EsSwgdGHFn8Sxbm1hesSxbiBla29ub21payBhbWFjxLFuYSB1eWd1biBvbGFyYWsgemlseWV0bGnEn2UgbmUgemFtYW4gYmHFn2xhbsSxbGTEscSfxLFuYSBpbGnFn2tpbiByYXBvciBkw7x6ZW5sZW5tZXNpIGlzdGVuaWxtacWfIG9sdXAgYnUga29udWxhcmEgaWxpxZ9raW4gMjAuMDUuMjAyNSB0YXJpaGxpIGJpbGlya2nFn2kgcmFwb3J1IGFsxLFubWFrIHZlIGJ1IHJhcG9yIGdlcmVrw6dlIGvEsXNtxLFuZGEgZGXEn2VybGVuZGlyaWxtZWsgc3VyZXRpeWxlIG1haGtlbWVjZSAgZGlyZW5tZSBrYXJhcsSxIHZlcmlsbWnFn3Rpci48YnI+Ny4gQnUgZHVydW1kYSDEsGxrIERlcmVjZSBNYWhrZW1lc2luaW4gZGlyZW5tZSBvbGFyYWsgYWRsYW5kxLFyZMSxxJ/EsSBrYXJhcsSxbiB1c3VsIGh1a3VrdSBhbmxhbcSxbmRhIGdlcsOnZWsgYmlyIGRpcmVubWUga2FyYXLEsSBvbG1hZMSxxJ/EsTsgaWxrIGthcmFyZGEgdGFydMSxxZ/EsWzEsXAsIGRlxJ9lcmxlbmRpcmlsbWVtacWfIHllbmkgYmlsaXJracWfaSByYXBvcnUgdmUgZ2VyZWvDp2V5ZSBkYXlhbMSxIHllbmkgaMO8a8O8bSBuaXRlbGnEn2luZGUgb2xkdcSfdSBrYWJ1bCBlZGlsbWVsaWRpci48YnI+OC4gSMOibCBiw7Z5bGUgb2x1bmNhIGt1cnVsYW4gYnUgeWVuaSBow7xrbcO8biB0ZW15aXogaW5jZWxlbm1lc2luaSB5YXBtYSBnw7ZyZXZpIEh1a3VrIEdlbmVsIEt1cnVsdW5hIGRlxJ9pbCwgw5Z6ZWwgRGFpcmV5ZSBhaXR0aXIuPGJyPjkuIEJ1IG5lZGVubGUsIHllbmkgaMO8a21lIHnDtm5lbGlrIHRlbXlpeiBpdGlyYXpsYXLEsW7EsW4gaW5jZWxlbm1lc2kgacOnaW4gZG9zeWFuxLFuIMOWemVsIERhaXJleWUgZ8O2bmRlcmlsbWVzaSBnZXJla2lyLjxicj48YnI+PGJyPktBUkFSPGJyPkHDp8Sxa2xhbmFuIHNlYmVwbGVybGU7PGJyPkRhdmFsxLEgdmVraWxpbmluIHllbmkgaMO8a21lIHnDtm5lbGlrIHRlbXlpeiBpdGlyYXpsYXLEsW7EsW4gaW5jZWxlbm1lc2kgacOnaW4gZG9zeWFuxLFuIFlBUkdJVEFZIDguIEhVS1VLIERBxLBSRVPEsE5FIEfDlk5ERVLEsExNRVPEsE5FLCA8YnI+MjMuMDkuMjAyNiB0YXJpaGluZGUgeWFwxLFsYW4gZ8O2csO8xZ9tZWRlIG95IGJpcmxpxJ9peWxlIGtlc2luIG9sYXJhayBrYXJhciB2ZXJpbGRpLjxicj48YnI+ICA8L2ZvbnQ+PC9wPjwvYm9keT48L2h0bWw+","mimeType":"text/html","version":1,"partType":"ICMBASETEXT","annotationPart":null},"metadata":{"FMTY":"SUCCESS","FMC":"ADALET_SUCCESS","FMTE":"İşlem başarıyla gerçekleştirildi!","FMU":"İşlem başarıyla gerçekleştirildi!","PTID":null,"TID":"735277f5ed915b4c3cad0af32da2158f","SID":"d419324e91c9766b"}} \ No newline at end of file diff --git a/tests/golden/bedesten/document-1231237100.json b/tests/golden/bedesten/document-1231237100.json new file mode 100644 index 0000000..38151e7 --- /dev/null +++ b/tests/golden/bedesten/document-1231237100.json @@ -0,0 +1 @@ +{"data":{"content":"PGh0bWw+PGhlYWQ+PG1ldGEgaHR0cC1lcXVpdj0iQ29udGVudC1UeXBlIiBjb250ZW50PSJ0ZXh0L2h0bWw7IGNoYXJzZXQ9VVRGLTgiPjwvaGVhZD4gPGJvZHkgbGVmdG1hcmdpbj0iMjUiIHRvcG1hcmdpbj0iMjAiIGZvbnQgZmFjZT0iVmVyZGFuYSIgc2l6ZT0iMiI+PGI+PGZvbnQgZmFjZT0iVmVyZGFuYSIgc2l6ZT0iMiI+SHVrdWsgR2VuZWwgS3VydWx1IMKgwqDCoMKgwqDCoMKgwqAyMDI2LzcxNCBFLiDCoCwgwqAyMDI2LzU3MyBLLjwvZm9udD48L2I+PHVsPjxsaSBzdHlsZT0iZm9udC1mYW1pbHk6VmVyZGFuYTtmb250LXNpemU6MTI7Zm9udC13ZWlnaHQ6Ym9sZCI+PC9saT48L3VsPjx1bCBzdHlsZT0ibGlzdC1zdHlsZS10eXBlOiBjaXJjbGU7Zm9udC1mYW1pbHk6VmVyZGFuYTtjb2xvcjojMTA0ZDk2O2ZvbnQtc2l6ZToxMiI+PC91bD48YnI+CjxiPjxmb250IGZhY2U9IlZlcmRhbmEiIHNpemU9IjIiPiLEsMOndGloYXQgTWV0bmkiPC9mb250PjwvYj48cCBhbGlnbj0ianVzdGlmeSI+PGZvbnQgZmFjZT0iVmVyZGFuYSIgc2l6ZT0iMiI+TUFIS0VNRVPEsAk6QXNsaXllIEh1a3VrIE1haGtlbWVzaTxicj5TQVlJU0kJOiAyMDI1LzIwOCBFc2FzLCAyMDI2LzQ1IEthcmFyPGJyPjxicj4JVGFyYWZsYXIgYXJhc8SxbmRha2kgYWxhY2FrIGRhdmFzxLFuZGFuIGRvbGF5xLEgeWFwxLFsYW4geWFyZ8SxbGFtYSBzb251bmRhLCBCYXlidXJ0IDEuIEFzbGl5ZSBIdWt1ayBNYWhrZW1lc2luY2UgdmVyaWxlbiBkaXJlbm1lIGthcmFyxLEgdGVteWl6IGVkaWxtZXNpIMO8emVyaW5lIEh1a3VrIEdlbmVsIEt1cnVsdW51biAyOC4wNS4yMDI1IHRhcmlobGkgdmUgMjAyMy82LTkwNiBFc2FzLCAyMDI1LzM2MSBLYXJhciBzYXnEsWzEsSBrYXJhcsSxIGlsZSBib3p1bG11xZ8sIHllbmlkZW4geWFwxLFsYW4geWFyZ8SxbGFtYSBzb251bmRhIG1haGtlbWVjZSBib3ptYXlhIHV5dWxhcmFrIGthcmFyIHZlcmlsbWnFn3Rpci48YnI+S2FyYXIgdGVteWl6IGVkaWxtacWfIHZlIGRvc3lhIHRla3JhciBIdWt1ayBHZW5lbCBLdXJ1bHVuYSBnw7ZuZGVyaWxtacWfdGlyLjxicj5IdWt1ayBHZW5lbCBLdXJ1bHVuY2EgZG9zeWEgw7x6ZXJpbmRlIHlhcMSxbGFuIMO2biBpbmNlbGVtZSBzb251bmRhIGdlcmXEn2kgZMO8xZ/DvG7DvGxkw7w6PGJyPk1haGtlbWVjZSB2ZXJpbGVuIGRpcmVubWUga2FyYXLEsSwgSHVrdWsgR2VuZWwgS3VydWx1bnVuIHl1a2FyxLFkYSB0YXJpaCB2ZSBzYXnEsXPEsSBiZWxpcnRpbGVuIGthcmFyxLEgaWxlIGJvenVsbXXFnzsgbWFoa2VtZWNlIGJvem1hIGlsw6JtxLFuYSB1eXVsYXJhayB5ZW5pIGJpciBrYXJhciB2ZXJpbG1pxZ90aXIuPGJyPkJ1IGR1cnVtZGEgZG9zeWEgw7x6ZXJpbmRlIEh1a3VrIEdlbmVsIEt1cnVsdSB0YXJhZsSxbmRhbiB5YXDEsWxhY2FrIGJpciBpxZ9sZW0gYnVsdW5tYW1ha3RhZMSxci4gPGJyPk8gaMOibGRlIHllbmkgaMO8a21lIHnDtm5lbGlrIHRlbXlpeiBpdGlyYXpsYXLEsSBpbmNlbGVubWVrIMO8emVyZSBkb3N5YSDDlnplbCBEYWlyZXllIGfDtm5kZXJpbG1lbGlkaXIuIDxicj5LQVJBUjxicj5Bw6fEsWtsYW5hbiBzZWJlcGxlcmxlOzxicj5NYWhrZW1lY2UgdmVyaWxlbiB5ZW5pIGjDvGttZSB5w7ZuZWxpayB0ZW15aXogaXRpcmF6bGFyxLFuxLFuIGluY2VsZW5tZXNpIGnDp2luIGRvc3lhbsSxbiBZQVJHSVRBWSA2LiBIVUtVSyBEQcSwUkVTxLBORSBHw5ZOREVSxLBMTUVTxLBORSwgPGJyPjAxLjEwLjIwMjYgdGFyaWhpbmRlIG95IGJpcmxpxJ9peWxlIGthcmFyIHZlcmlsZGkuPGJyPiAgPGJyPjwvZm9udD48L3A+PC9ib2R5PjwvaHRtbD4=","mimeType":"text/html","version":1,"partType":"ICMBASETEXT","annotationPart":null},"metadata":{"FMTY":"SUCCESS","FMC":"ADALET_SUCCESS","FMTE":"İşlem başarıyla gerçekleştirildi!","FMU":"İşlem başarıyla gerçekleştirildi!","PTID":null,"TID":"382a474ced0c7441709458dbf50ff2e2","SID":"ca3894122ecb5441"}} \ No newline at end of file diff --git a/tests/golden/bedesten/search.json b/tests/golden/bedesten/search.json new file mode 100644 index 0000000..d9c7956 --- /dev/null +++ b/tests/golden/bedesten/search.json @@ -0,0 +1 @@ +{"data":{"emsalKararList":[{"documentId":"1231237100","itemType":{"name":"YARGITAYKARARI","description":"Yargıtay Kararı"},"birimId":null,"birimAdi":"Hukuk Genel Kurulu","esasNoYil":2026,"esasNoSira":714,"kararNoYil":2026,"kararNoSira":573,"kararTuru":null,"kararTarihi":"2026-09-30T21:00:00.000+00:00","kararTarihiStr":"01.10.2026","kesinlesmeDurumu":null,"kararNo":"2026/573","esasNo":"2026/714"},{"documentId":"1231234400","itemType":{"name":"YARGITAYKARARI","description":"Yargıtay Kararı"},"birimId":null,"birimAdi":"Hukuk Genel Kurulu","esasNoYil":2025,"esasNoSira":510,"kararNoYil":2026,"kararNoSira":529,"kararTuru":null,"kararTarihi":"2026-09-22T21:00:00.000+00:00","kararTarihiStr":"23.09.2026","kesinlesmeDurumu":null,"kararNo":"2026/529","esasNo":"2025/510"},{"documentId":"1231234100","itemType":{"name":"YARGITAYKARARI","description":"Yargıtay Kararı"},"birimId":null,"birimAdi":"Hukuk Genel Kurulu","esasNoYil":2026,"esasNoSira":672,"kararNoYil":2026,"kararNoSira":539,"kararTuru":null,"kararTarihi":"2026-09-22T21:00:00.000+00:00","kararTarihiStr":"23.09.2026","kesinlesmeDurumu":null,"kararNo":"2026/539","esasNo":"2026/672"}],"total":7398,"start":0},"metadata":{"FMTY":"SUCCESS","FMC":"ADALET_SUCCESS","FMTE":"İşlem başarıyla gerçekleştirildi!","FMU":"İşlem başarıyla gerçekleştirildi!","PTID":null,"TID":"41c8a56dd017508f73c612aa07a73623","SID":"0cd95a8a93f14e1c"}} \ No newline at end of file diff --git a/tests/golden/resmi_gazete/20260809.htm b/tests/golden/resmi_gazete/20260809.htm new file mode 100644 index 0000000..6a4274b --- /dev/null +++ b/tests/golden/resmi_gazete/20260809.htm @@ -0,0 +1,315 @@ + + + + + + + +T.C. Resm Gazete + + + + + + + + + +
+ + + + + + + + + +
+

+ 9 + Austos 2026 Tarihli ve 33335 Sayl Resm + Gazete

 +

+ MEVZUAT

+ + + + + + +
+   +

+ +

+

 

+
+

+ +YASAMA BLM

+

 

+

+ + +KANUNLAR

+

+ +7591  +Trkiye Cumhuriyeti Hkmeti ile Birlemi Milletler klim Deiiklii ereve +Szlemesi, Kyoto Protokol ve Paris Anlamas Sekretaryas Arasnda Birlemi +Milletler klim Deiiklii ereve Szlemesi Taraflar Konferansnn Otuz +Birinci Oturumu, Kyoto Protokol Taraflar Toplants Olarak Dzenlenen Taraflar +Konferansnn Yirmi Birinci Oturumu, Paris Anlamas Taraflar Toplants Olarak +Dzenlenen Taraflar Konferansnn Sekizinci Oturumu, Bal Organlarn Oturumlar +ve Dier Bmids Toplantlarna Dair Anlamann Onaylanmasnn Uygun Bulunduuna +Dair Kanun

+

+ +7592  Yksekretim +Kanunu ve Baz Kanunlarda Deiiklik Yaplmasna Dair Kanun

+

 

+

+ +YRTME VE DARE BLM

+

 

+

+ + +CUMHURBAKANLII KARARNAMELER

+

+ +  Emniyet Genel +Mdrlne Kadro hdas Edilmesine likin Cumhurbakanl Kararnamesi +(Kararname Numaras: 199)

+

+ +  Genel Kadro ve Usul +Hakknda Cumhurbakanl Kararnamesinde Deiiklik Yaplmasna Dair +Cumhurbakanl Kararnamesi (Kararname Numaras: 200)

+

 

+

+ + +CUMHURBAKANI KARARLARI

+

+ +  Kamu Kurum ve +Kurulularnn Yurtd Tekilatn Oluturan Birimlerin Nitelik, Kurulduu +ehir ve lke, Grev Alan, Akredite Edildii lkeler ve Bal Bulunduu +Misyonlar Hakkndaki 13/4/1999 Tarihli ve 99/12770 Sayl Bakanlar Kurulu +Kararnda Deiiklik Yaplmasna likin Karar (Karar Says: 11569)

+

+ +  Kamu Kurum ve +Kurulularnn Yurtd Tekilatn Oluturan Birimlerin Nitelik, Kurulduu +ehir ve lke, Grev Alan, Akredite Edildii lkeler ve Bal Bulunduu +Misyonlar Hakkndaki 13/4/1999 Tarihli ve 99/12770 Sayl Bakanlar Kurulu +Kararnda Deiiklik Yaplmasna likin Karar (Karar Says: 11570)

+

+ +  Orman Genel +Mdrlnn Tara Tekilatnda Belirtilen Orman letme Mdrlnn Kurulmas +Hakknda Karar (Karar Says: 11571)

+

+ +  Yatrmlara Proje +Bazl Devlet Yardm Verilmesine likin Kararda Deiiklik Yaplmasna Dair +Karar (Karar Says: 11572)

+

+ +  Belediyeler ve l +zel darelerinin Genel Bte Vergi Gelirleri Payndan Yaplacak Aydnlatma +Gideri Kesintilerine likin Karar (Karar Says: 11573)

+

+ +  Mula li, Kyceiz, +Milas, Seydikemer ve Yataan lelerinde Meydana Gelen Ar Yalar Sonucunda +Baz Kprlerin Tayc Ayaklar ile Zeminlerinde Oluan Kalc Hasar Nedeniyle +Ortaya kan Can ve Mal Gvenlii Riskinin Bertaraf Edilmesi Amacyla Baz +Tanmazlarn, Mula Bykehir Belediye Bakanl Tarafndan Acele +Kamulatrlmas Hakknda Karar (Karar Says: 11574)

+

+ +  anlurfa li, +Eyybiye lesi Snrlar erisinde Bulunan Tanmazlar ile zerlerindeki +Muhdesatn, Eyybiye-Haliliye Hafif Rayl Sistem 1. Aama Hatt ile anlurfa +Eitim ve Aratrma Hastanesi n Katl Kavak Projesi Kapsamnda anlurfa +Bykehir Belediye Bakanl Tarafndan Acele Kamulatrlmas Hakknda Karar  +(Karar Says: 11575)

+

+ +  154 kV Falp RES +TM-ine TM Enerji letim Hatt Projesi Kapsamnda Gsterilen Baz Tanmazlarn +Trkiye Elektrik letim Anonim irketi Genel Mdrl Tarafndan Acele +Kamulatrlmas Hakknda Karar  (Karar Says: 11576)

+

+ +  400 kV Glba +Kayseri Kapasitr Kuzey Enerji letim Hatt Ksmi Yenileme Projesi Kapsamnda +Gsterilen Baz Tanmazlarn Trkiye Elektrik letim Anonim irketi Genel +Mdrl Tarafndan Acele Kamulatrlmas Hakknda Karar  (Karar Says: +11577)

+

+ +  anlurfa li, +Eyybiye lesi, Batkent Mahallesi Snrlar erisinde Bulunan Tanmazlar ile +zerlerindeki Muhdesatn, Evren Sanayi Sitesi Kprl Kavak Projesi Kapsamnda +anlurfa Bykehir Belediye Bakanl Tarafndan Acele Kamulatrlmas +Hakknda Karar  (Karar Says: 11578)

+

+ +  Erzincan li, Merkez +lesi, Karaaa Mahallesi Snrlar erisinde Bulunan Baz  Tanmazlarn, +Blgede Yrtlen Kentsel Dnm ve Geliim Projesi Kapsamnda Erzincan +Belediye Bakanl Tarafndan Acele Kamulatrlmas Hakknda Karar  (Karar +Says: 11579)

+

+ +  400/33 kV Bakent +OSB Transformatr Merkezi Sahas Projesi Kapsamnda Baz Tanmazlarn Trkiye +Elektrik letim Anonim irketi Genel Mdrl Tarafndan Acele Kamulatrlmas +Hakknda Karar  (Karar Says: 11580)

+

+ +  zelletirme daresi +Bakanl ile lgili Kararlar (Karar Says: 11581, 11582, 11583, 11584)

+

 

+

+ + +ATAMA KARARI

+

+ +  +Cumhurbakanl Tarafndan Yaplan Atama Hakknda Karar (Karar: 2026/223)

+

 

+

+ + +YNETMELKLER

+

+ +  Hazine +Tanmazlarnn daresi Hakknda Ynetmelikte Deiiklik Yaplmasna Dair +Ynetmelik

+

+ +  +stanbul Yeni Yzyl niversitesi Trke retimi Uygulama ve Aratrma Merkezi +Ynetmeliinde Deiiklik Yaplmasna Dair Ynetmelik

+

+ +  zmir +Tnaztepe niversitesi Tp Fakltesi Eitim retim ve Snav Ynetmeliinde +Deiiklik Yaplmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Avrupa Birlii Aratrma ve Uygulama Merkezi (SABAUM) +Ynetmeliinin Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Doal rnler Uygulama ve Aratrma Merkezi Ynetmeliinin +Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Dnya Kltr ve Medeniyetleri Uygulama ve Aratrma Merkezi +Ynetmeliinin Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Gvenlik ve Savunma Stratejileri Uygulama ve Aratrma Merkezi +(GSSAGEM) Ynetmeliinin Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Havaclk ve Uzay Teknolojileri Uygulama ve Aratrma Merkezi +Ynetmeliinin Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Sal ve Gvenlii, Meslek Hastalklar Uygulama ve Aratrma +Merkezi Ynetmeliinin Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Kazalar Aratrma, nleme ve Uygulama Merkezi Ynetmeliinin +Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Konya ve Yresi Tarih Aratrmalar Merkezi Ynetmeliinin +Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Kkba Hayvanclk Uygulama ve Aratrma Merkezi Ynetmeliinin +Yrrlkten Kaldrlmasna Dair Ynetmelik

+

+ +  Seluk +niversitesi Nasreddin Hoca Uygulama ve Aratrma Merkezi Ynetmeliinin +Yrrlkten Kaldrlmasna Dair Ynetmelik

+

 

+

LN BLM

+

+ + + a - Artrma, Eksiltme ve + hale lnlar

+

+ + + b - eitli lnlar

+

+ + T.C. Merkez Bankasnca + Belirlenen Devlet Borlanma Senetlerinin Gnlk + Deerleri

+

+ 

+
+

+ +Resm Gazete'nin kurumsal mobil uygulamas Android ve Apple marketlerde "T.C. Resm Gazete" adyla yerini almtr.

+
+
+ + + + \ No newline at end of file diff --git a/tests/golden/resmi_gazete/20260812.htm b/tests/golden/resmi_gazete/20260812.htm new file mode 100644 index 0000000..419e503 --- /dev/null +++ b/tests/golden/resmi_gazete/20260812.htm @@ -0,0 +1,148 @@ + + + + + + + +T.C. Resm Gazete + + + + + + + + + +
+ + + + + + + + + +
+

+ 12 + Austos 2026 Tarihli ve 33338 Sayl Resm + Gazete

 +

+ MEVZUAT

+ + + + + + +
+   +

+ +

+

 

+
+

+YASAMA +BLM

+

 

+

+ +TBMM KARARI

+

+ +1512 Trkiye Byk Millet +Meclisinin Tatile Girmesine likin Karar

+

 

+

+YRTME +VE DARE BLM

+

 

+

+ +YNETMELK

+

+ +  Binalarn Yklmas +Hakknda Ynetmelikte Deiiklik Yaplmasna Dair Ynetmelik

+

 

+

+ +TEBLLER

+

+ +  Proje ve Kontrolluk +lerinde Uygulanacak Fiyat Art Oranlar Hakknda Tebli

+

+ +  Tarm ve Orman +Bakanl Kontrolne Tabi Belirli rnlerin Giriine Yetkili Gmrk dareleri +ile Resmi Kontrollerini Yapmaya Yetkili l Tarm ve Orman Mdrlklerinin +Belirlenmesine Dair Teblide Deiiklik Yaplmasna Dair Tebli

+

 

+

LN BLM

+

+ a - Yarg lnlar

+

+ b - Artrma, Eksiltme ve + hale lnlar

+

+ c - eitli lnlar

+

+ + T.C. Merkez Bankasnca + Belirlenen Dviz Kurlar ve Devlet Borlanma Senetlerinin Gnlk + Deerleri

+

+ 

+
+

+ +Resm Gazete'nin kurumsal mobil uygulamas Android ve Apple marketlerde "T.C. Resm Gazete" adyla yerini almtr.

+
+
+ + + + \ No newline at end of file diff --git a/tests/golden/resmi_gazete/20260813.htm b/tests/golden/resmi_gazete/20260813.htm new file mode 100644 index 0000000..628013b --- /dev/null +++ b/tests/golden/resmi_gazete/20260813.htm @@ -0,0 +1,147 @@ + + + + + + + +T.C. Resm Gazete + + + + + + + + + +
+ + + + + + + + + +
+

+ 13 + Austos 2026 Tarihli ve 33339 Sayl Resm + Gazete

 +

+ MEVZUAT

+ + + + + + +
+   +

+ +

+

 

+
+

+YRTME +VE DARE BLM

+

 

+

+ +CUMHURBAKANI KARARI

+

+ +  Baz Mallara +Uygulanan zel Tketim Vergisi Tutarlarnn Yeniden Belirlenmesi Hakknda Karar +(Karar Says: 11606)

+

 

+

+ +YNETMELKLER

+

+ +  mrn Tamamlam +Lastiklerin Kontrol Ynetmeliinde Deiiklik Yaplmasna Dair Ynetmelik

+

+ +  Boazii niversitesi +Lisans Eitim-retim Ynetmeliinde Deiiklik Yaplmasna Dair Ynetmelik

+

 

+

+ +KURUL KARARLARI

+

+ +  Kiisel Verileri +Koruma Kurulunun 22/07/2026 Tarihli ve 2026/1491 Sayl Karar

+

+ +  Nkleer Dzenleme +Kurulunun 6/8/2026 Tarihli ve 2026-34/2 Sayl Karar

+

 

+

LN BLM

+

+ a - Yarg lnlar

+

+ b - Artrma, Eksiltme ve + hale lnlar

+

+ c - eitli lnlar

+

+ + T.C. Merkez Bankasnca + Belirlenen Dviz Kurlar ve Devlet Borlanma Senetlerinin Gnlk + Deerleri

+

+ 

+
+

+ +Resm Gazete'nin kurumsal mobil uygulamas Android ve Apple marketlerde "T.C. Resm Gazete" adyla yerini almtr.

+
+
+ + + + \ No newline at end of file diff --git a/tests/integration/test_collect_url.py b/tests/integration/test_collect_url.py index e38230c..c8ed1ba 100644 --- a/tests/integration/test_collect_url.py +++ b/tests/integration/test_collect_url.py @@ -14,13 +14,30 @@ def test_collect_url_disabled_source_fails(tmp_path, monkeypatch): db_path = get_db_path() migrate(None, db_path) - # Yargitay is disabled in sources.yaml + sources_yaml_path = tmp_path / "sources.yaml" + sources_yaml_path.write_text( + 'version: "1.0.0"\n' + "sources:\n" + " yargitay:\n" + " name: Disabled Yargitay\n" + " authority: Yargitay\n" + " base_url: https://karararama.yargitay.gov.tr/\n" + " access_mode: manual\n" + " enabled: false\n" + " families: [decision]\n" + " source_role: official_case_law\n" + ' policy_version: "1.0.0"\n' + " http: {user_agent: MESA-Legal-Data-Test/1.0}\n" + " allowed_content_types: [text/html]\n", + encoding="utf-8", + ) with pytest.raises(SourcePolicyError, match="SOURCE_DISABLED"): import_manual_url( url="https://karararama.yargitay.gov.tr/doc.html", source_id="yargitay", document_id="tr:case-law:yargitay:2026:1", family="decision", + sources_yaml_path=sources_yaml_path, ) diff --git a/tests/integration/test_master_mvp_closure.py b/tests/integration/test_master_mvp_closure.py index 1db753b..df51d5b 100644 --- a/tests/integration/test_master_mvp_closure.py +++ b/tests/integration/test_master_mvp_closure.py @@ -31,7 +31,7 @@ from mesa_legal_data.harvest.service_bridge import run_pipeline_item from mesa_legal_data.hashing import hash_stream from mesa_legal_data.pipeline import process_artifact_pipeline -from mesa_legal_data.publisher.client import MesaClientError +from mesa_legal_data.publisher.client import MesaClient, MesaClientError from mesa_legal_data.publisher.engine import build_delivery_plan, execute_publish_delivery from mesa_legal_data.publisher.ledger import upsert_mesa_target_settings from mesa_legal_data.publisher.models import DeliveryStatus, MesaTargetSettings @@ -199,21 +199,18 @@ def fake_submit(conn, *, operation_type, requested_by, input_dict): frozen_chunks, _ = build_delivery_plan(release_id=captured["release_id"]) assert {chunk.version_id for chunk, _ in frozen_chunks} == {v1["version_id"]} citation_metadata = [citation for chunk, _ in frozen_chunks for citation in chunk.metadata.get("citations", [])] - assert citation_metadata == [ - { - "citation_id": citation_metadata[0]["citation_id"], - "source_record_id": doc_id, - "source_version_id": v1["version_id"], - "target_legislation_id": "tr:legislation:law:6098", - "target_article_id": "tr:legislation:law:6098:article:1-a", - "target_version_id": None, - "citation_status": "EXTRACTED", - "relation_hint": "amends", - "char_start": citation_metadata[0]["char_start"], - "char_end": citation_metadata[0]["char_end"], - "fully_contained": True, - } - ] + assert citation_metadata == ["tr:legislation:law:6098:article:1-a"] + # MESA rejects metadata with objects or lists inside lists, and metadata over 16 KB. + client = MesaClient(MesaTargetSettings(), api_key="secret") + for chunk, key in frozen_chunks: + metadata = client.build_memory_insert_payload( + chunk, session_id="sess-1", idempotency_key=key, finalize_revision=False + )["metadata"] + assert len(json.dumps(metadata, sort_keys=True, default=str).encode()) <= 16 * 1024 + for value in metadata.values(): + assert not isinstance(value, dict) + if isinstance(value, list): + assert all(not isinstance(item, (dict, list)) for item in value) def test_master_a_chronology_current_version_selection(tmp_path, monkeypatch): diff --git a/tests/integration/test_mesa_v4_publisher_e2e.py b/tests/integration/test_mesa_v4_publisher_e2e.py index bbb72ab..675f116 100644 --- a/tests/integration/test_mesa_v4_publisher_e2e.py +++ b/tests/integration/test_mesa_v4_publisher_e2e.py @@ -255,3 +255,182 @@ def response_loss_then_commit(request): assert retried["retried_count"] == 1 assert len(seen_keys) == 3 assert seen_keys[0] == seen_keys[2] + + +@pytest.fixture +def cli_verified_release(tmp_path, monkeypatch): + from mesa_legal_data.catalog import approve_version_with_checks, get_release, get_version_for_artifact + from mesa_legal_data.pipeline import process_artifact_pipeline + from mesa_legal_data.publisher.engine import target_config_sha256 + from mesa_legal_data.release.builder import build_release + from mesa_legal_data.release.verifier import verify_release + from mesa_legal_data.sources.manual import import_manual_file + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_MESA_API_KEY", "test-secret") + migrate() + source = tmp_path / "tbk.html" + source.write_text( + "

6098 TÜRK BORÇLAR KANUNU

" + "

MADDE 1- Sözleşme, tarafların iradelerini karşılıklı ve birbirine uygun olarak açıklamalarıyla kurulur.

" + "", + encoding="utf-8", + ) + artifact = import_manual_file( + source, + source_id="mevzuat", + document_id="tr:legislation:law:6098", + extra_metadata={"snapshot_date": "2026-10-27"}, + ) + process_artifact_pipeline(artifact.artifact_id) + conn = get_connection() + version = get_version_for_artifact(conn, artifact.artifact_id) + approve_version_with_checks(conn, version["version_id"], reviewer="test-human") + settings = MesaTargetSettings(base_url="http://127.0.0.1:8000", contract_source="configured") + upsert_mesa_target_settings(conn, settings) + conn.close() + build_release("cli-release") + assert verify_release("cli-release") + conn = get_connection() + release = get_release(conn, "cli-release") + conn.close() + return tmp_path, release, target_config_sha256(settings) + + +@respx.mock +def test_cli_publish_verified_release_and_repeat(cli_verified_release): + from typer.testing import CliRunner + + from mesa_legal_data.cli import app + + _, release, target_hash = cli_verified_release + base = "http://127.0.0.1:8000" + respx.get(base + "/health").respond(200, json={"status": "ok"}) + starts = respx.post(base + "/v4/sessions/start").respond(201, json={"session_id": "cli-session"}) + inserts = respx.post(base + "/v4/memory/insert").respond( + 202, json={"mutation_id": "cli-mutation", "status": "accepted"} + ) + respx.get(base + "/v4/mutations/cli-mutation").respond(200, json={"state": "COMMITTED"}) + respx.post(base + "/v4/sessions/cli-session/end").respond(200, json={"status": "ended"}) + args = [ + "publish", + "run", + "--release-id", + release["release_id"], + "--confirm-manifest", + release["manifest_sha256"], + "--confirm-target", + target_hash, + ] + runner = CliRunner() + result = runner.invoke(app, args) + assert result.exit_code == 0, result.output + assert "COMMITTED" in result.output + assert inserts.call_count == 2 # Heading preamble and one numbered article. + payloads = [json.loads(call.request.content) for call in inserts.calls] + assert [payload["metadata"]["mesa_data_chunk_type"] for payload in payloads] == ["preamble", "article"] + assert "legal_identity" not in payloads[0] + assert payloads[1]["legal_identity"] == {"law_id": "6098", "article_id": "1"} + conn = get_connection() + delivery_id, status = conn.execute("SELECT delivery_id, status FROM mesa_deliveries").fetchone() + assert status == "COMMITTED" + assert delivery_id in result.output + events = conn.execute( + "SELECT actor, subject_id, new_sha256, details_json FROM audit_events WHERE action = 'mesa_publish_authorize'" + ).fetchall() + assert len(events) == 1 + assert events[0][0] == "cli" + assert events[0][1:3] == (release["release_id"], release["manifest_sha256"]) + details = json.loads(events[0][3]) + assert details["delivery_id"] == delivery_id + assert details["target_config_sha256"] == target_hash + assert details["scope"]["ready_versions"] == 1 + assert conn.execute("SELECT count(*) FROM operation_jobs").fetchone()[0] == 0 + conn.close() + status_result = runner.invoke(app, ["publish", "status", delivery_id]) + assert status_result.exit_code == 0, status_result.output + assert "COMMITTED" in status_result.output and delivery_id in status_result.output + repeated = runner.invoke(app, args) + assert repeated.exit_code == 0, repeated.output + assert inserts.call_count == 2 + assert starts.call_count == 1 + + +@pytest.mark.parametrize( + "scenario, error", + [ + ("manifest", "RELEASE_CONFIRMATION_MISMATCH"), + ("target", "TARGET_CONFIRMATION_MISMATCH"), + ("unverified", "RELEASE_STATUS_INVALID"), + ("missing_release", "RELEASE_NOT_FOUND"), + ("missing_confirmation", "Missing option"), + ("tamper", "File hash mismatch"), + ("preflight", "PREFLIGHT_FAILED"), + ], +) +@respx.mock +def test_cli_publish_rejects_before_sending(cli_verified_release, scenario, error): + from typer.testing import CliRunner + + from mesa_legal_data.cli import app + + root, release, target_hash = cli_verified_release + manifest = release["manifest_sha256"] + release_id = release["release_id"] + if scenario == "manifest": + manifest = "0" * 64 + elif scenario == "target": + target_hash = "0" * 64 + elif scenario == "unverified": + conn = get_connection() + conn.execute("UPDATE releases SET status = 'preparing' WHERE release_id = ?", (release_id,)) + conn.close() + elif scenario == "missing_release": + release_id = "missing-release" + elif scenario == "tamper": + (root / "releases" / release_id / "data" / "articles.jsonl").write_text("{}\n") + respx.get("http://127.0.0.1:8000/health").respond(503 if scenario == "preflight" else 200, json={"status": "ok"}) + args = ["publish", "run", "--release-id", release_id, "--confirm-manifest", manifest] + if scenario != "missing_confirmation": + args += ["--confirm-target", target_hash] + result = CliRunner().invoke(app, args) + assert result.exit_code != 0 + assert error.lower() in result.output.lower(), result.output + assert not any(call.request.method == "POST" for call in respx.calls) + conn = get_connection() + assert conn.execute("SELECT count(*) FROM mesa_deliveries").fetchone()[0] == 0 + assert conn.execute("SELECT count(*) FROM audit_events WHERE action = 'mesa_publish_authorize'").fetchone()[0] == 0 + conn.close() + + +@respx.mock +def test_cli_publish_exits_nonzero_unless_committed(cli_verified_release): + from typer.testing import CliRunner + + from mesa_legal_data.cli import app + + _, release, target_hash = cli_verified_release + base = "http://127.0.0.1:8000" + respx.get(base + "/health").respond(200, json={"status": "ok"}) + respx.post(base + "/v4/sessions/start").respond(201, json={"session_id": "cli-session"}) + respx.post(base + "/v4/memory/insert").respond(202, json={"mutation_id": "cli-mutation", "status": "accepted"}) + respx.get(base + "/v4/mutations/cli-mutation").respond(200, json={"state": "REJECTED"}) + result = CliRunner().invoke( + app, + [ + "publish", + "run", + "--release-id", + release["release_id"], + "--confirm-manifest", + release["manifest_sha256"], + "--confirm-target", + target_hash, + ], + ) + assert result.exit_code != 0 + assert "FAILED" in result.output or "PARTIAL" in result.output, result.output + conn = get_connection() + delivery = conn.execute("SELECT delivery_id, status FROM mesa_deliveries").fetchone() + assert delivery[0] in result.output + conn.close() diff --git a/tests/unit/harvest/test_resmi_gazete_adapter.py b/tests/unit/harvest/test_resmi_gazete_adapter.py index 3002e0f..35c4838 100644 --- a/tests/unit/harvest/test_resmi_gazete_adapter.py +++ b/tests/unit/harvest/test_resmi_gazete_adapter.py @@ -1,4 +1,6 @@ +import re from datetime import date +from pathlib import Path import pytest @@ -6,6 +8,7 @@ DiscoveryStructureChangedError, ResmiGazeteDiscoveryAdapter, ) +from mesa_legal_data.parsers.encoding import decode_source_bytes SAMPLE_RESMI_GAZETE_HTML = """ @@ -61,3 +64,102 @@ def test_resmi_gazete_adapter_structure_changed(): pub_date = date(2026, 8, 7) with pytest.raises(DiscoveryStructureChangedError): adapter.parse_html_fihrist("", pub_date, "https://www.resmigazete.gov.tr/empty") + + +def test_golden_resmi_gazete_20260812(): + golden_path = Path(__file__).resolve().parents[2] / "golden" / "resmi_gazete" / "20260812.htm" + html_content, _ = decode_source_bytes(golden_path.read_bytes(), is_html=True) + adapter = ResmiGazeteDiscoveryAdapter() + pub_date = date(2026, 8, 12) + docs = adapter.parse_html_fihrist( + html_content, pub_date, "https://www.resmigazete.gov.tr/eskiler/2026/08/20260812.htm" + ) + + # 1. Regulation link 20260812-2.htm -> type regulation + reg_docs = [d for d in docs if "20260812-2.htm" in d.document_url] + assert len(reg_docs) == 1 + reg_doc = reg_docs[0] + assert reg_doc.document_type == "regulation" + assert reg_doc.family == "legislation" + assert "Binaların Yıkılması" in reg_doc.title + + # 2. Every title has no \r, \n, \xa0, control chars, or leading dashes + for d in docs: + assert "\r" not in d.title + assert "\n" not in d.title + assert "\xa0" not in d.title + assert not re.search(r"[\x00-\x1f\x7f-\x9f]", d.title) + assert not d.title.startswith("–") + assert not d.title.startswith("-") + + # 3. No link in an İLAN / İLÂN section is returned + for d in docs: + assert "İLAN" not in d.section.upper() + assert "İLÂN" not in d.section.upper() + assert "ilanlar" not in d.document_url + + +def test_golden_resmi_gazete_20260813(): + golden_path = Path(__file__).resolve().parents[2] / "golden" / "resmi_gazete" / "20260813.htm" + html_content, _ = decode_source_bytes(golden_path.read_bytes(), is_html=True) + adapter = ResmiGazeteDiscoveryAdapter() + pub_date = date(2026, 8, 13) + docs = adapter.parse_html_fihrist( + html_content, pub_date, "https://www.resmigazete.gov.tr/eskiler/2026/08/20260813.htm" + ) + + # Decision link 20260813-0.pdf -> type presidential_decision, document_id tr:legislation:presidential_decision:11606 + dec_docs = [d for d in docs if "20260813-0.pdf" in d.document_url] + assert len(dec_docs) == 1 + dec_doc = dec_docs[0] + assert dec_doc.document_type == "presidential_decision" + assert dec_doc.document_id == "tr:legislation:presidential_decision:11606" + assert "Özel Tüketim Vergisi" in dec_doc.title + + for d in docs: + assert "\r" not in d.title + assert "\n" not in d.title + assert "\xa0" not in d.title + assert not re.search(r"[\x00-\x1f\x7f-\x9f]", d.title) + assert not d.title.startswith("–") + assert not d.title.startswith("-") + assert "İLAN" not in d.section.upper() + assert "İLÂN" not in d.section.upper() + + +def test_golden_resmi_gazete_20260809_laws_and_decrees(): + golden_path = Path(__file__).resolve().parents[2] / "golden" / "resmi_gazete" / "20260809.htm" + html_content, _ = decode_source_bytes(golden_path.read_bytes(), is_html=True) + adapter = ResmiGazeteDiscoveryAdapter() + pub_date = date(2026, 8, 9) + docs = adapter.parse_html_fihrist( + html_content, pub_date, "https://www.resmigazete.gov.tr/eskiler/2026/08/20260809.htm" + ) + + # Law links -> type law, document_id tr:legislation:law:, title keeps its number + law7592 = [d for d in docs if "20260809-2.htm" in d.document_url] + assert len(law7592) == 1 + assert law7592[0].document_type == "law" + assert law7592[0].document_id == "tr:legislation:law:7592" + assert law7592[0].title.startswith("7592") + assert "Yükseköğretim Kanunu" in law7592[0].title + + # Presidential decree -> kararname number + decree199 = [d for d in docs if "20260809-3.pdf" in d.document_url] + assert len(decree199) == 1 + assert decree199[0].document_type == "presidential_decree" + assert decree199[0].document_id == "tr:legislation:presidential_decree:199" + + # Presidential decision -> Karar Sayısı + decision11577 = [d for d in docs if "20260809-13.pdf" in d.document_url] + assert len(decision11577) == 1 + assert decision11577[0].document_type == "presidential_decision" + assert decision11577[0].document_id == "tr:legislation:presidential_decision:11577" + + for d in docs: + assert "\r" not in d.title + assert "\n" not in d.title + assert "\xa0" not in d.title + assert not re.search(r"[\x00-\x1f\x7f-\x9f]", d.title) + assert "İLAN" not in d.section.upper() + assert "İLÂN" not in d.section.upper() diff --git a/tests/unit/test_bedesten.py b/tests/unit/test_bedesten.py new file mode 100644 index 0000000..d34a4a9 --- /dev/null +++ b/tests/unit/test_bedesten.py @@ -0,0 +1,171 @@ +import base64 +import json +from datetime import date +from pathlib import Path + +import httpx +import pytest +import respx +from typer.testing import CliRunner + +from mesa_legal_data.catalog import get_connection, migrate +from mesa_legal_data.cli import app +from mesa_legal_data.parsers.decision import parse_decision_text +from mesa_legal_data.sources import request_control +from mesa_legal_data.sources.url_fetcher import URLFetchError + +FIXTURES = Path(__file__).parents[1] / "golden" / "bedesten" +BASE_URL = "https://bedesten.adalet.gov.tr/emsal-karar/" + + +def fixture_json(name): + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +@respx.mock +def test_bedesten_search_import_pipeline_and_duplicate(tmp_path, monkeypatch): + from mesa_legal_data.sources.bedesten import collect_bedesten + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_OPERATOR_CONTACT", "operator@mesa.test") + migrate() + search = fixture_json("search.json") + search["data"]["emsalKararList"] = search["data"]["emsalKararList"][:2] + times = [] + + def search_response(request): + payload = json.loads(request.content) + assert payload["data"]["birimAdi"] == "Hukuk Genel Kurulu" + assert payload["data"]["phrase"] == "Hukuk Genel Kurulu" + assert payload["data"]["kararTarihiStart"] == "2021-01-01T00:00:00" + assert payload["data"]["kararTarihiEnd"] == "2026-10-27T23:59:59" + assert payload["data"]["pageSize"] == 2 + assert request.headers["AdaletApplicationName"] == "UyapMevzuat" + assert "operator@mesa.test" in request.headers["User-Agent"] + times.append(request_control.time_func()) + return httpx.Response(200, json=search) + + def content_response(request): + document_id = json.loads(request.content)["data"]["documentId"] + data = fixture_json(f"document-{document_id}.json") + # Conflicting text must not override the source's structured metadata. + text = base64.b64decode(data["data"]["content"]).decode("utf-8") + text += "

Esas No: 1999/1 Karar No: 1999/2 Karar Tarihi: 01.01.1999

" + data["data"]["content"] = base64.b64encode(text.encode()).decode() + times.append(request_control.time_func()) + return httpx.Response(200, json=data) + + respx.post(BASE_URL + "searchDocuments").mock(side_effect=search_response) + respx.post(BASE_URL + "getDocumentContent").mock(side_effect=content_response) + artifacts = collect_bedesten("HGK", date(2021, 1, 1), date(2026, 10, 27), limit=2) + assert len(artifacts) == 2 + conn = get_connection() + records = conn.execute( + "SELECT canonical_path, canonical_line FROM records WHERE record_type = 'decision' ORDER BY record_id" + ).fetchall() + assert len(records) == 2 + parsed_records = [ + json.loads((tmp_path / row[0]).read_text(encoding="utf-8").splitlines()[row[1] - 1]) for row in records + ] + by_esas = {record["esas_no"]: record for record in parsed_records} + for item in search["data"]["emsalKararList"]: + record = by_esas[item["esasNo"]] + assert record["id"] == ( + "tr:case-law:yargitay:hukuk-genel-kurulu:" + + item["esasNo"].replace("/", "-") + + ":" + + item["kararNo"].replace("/", "-") + ) + assert record["court"] == "YARGITAY" + assert record["chamber"] == "Hukuk Genel Kurulu" + assert record["karar_no"] == item["kararNo"] + assert ( + record["decision_date"] + == date.fromisoformat("-".join(reversed(item["kararTarihiStr"].split(".")))).isoformat() + ) + version_dates = {row[0] for row in conn.execute("SELECT snapshot_date FROM versions")} + assert version_dates == {"2026-10-01", "2026-09-23"} + before = [ + conn.execute(f"SELECT count(*) FROM {table}").fetchone()[0] for table in ("artifacts", "versions", "records") + ] + conn.close() + repeated = collect_bedesten("HGK", date(2021, 1, 1), date(2026, 10, 27), limit=2) + assert all(artifact.is_duplicate for artifact in repeated) + conn = get_connection() + assert [ + conn.execute(f"SELECT count(*) FROM {table}").fetchone()[0] for table in ("artifacts", "versions", "records") + ] == before + conn.close() + assert all(b - a >= 6 for a, b in zip(times, times[1:])) + + +@pytest.mark.parametrize("retry_succeeds", [True, False]) +@respx.mock +def test_bedesten_429_waits_and_retries_once(tmp_path, monkeypatch, retry_succeeds): + from mesa_legal_data.sources.bedesten import collect_bedesten + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_OPERATOR_CONTACT", "operator@mesa.test") + migrate() + times = [] + + def response(request): + times.append(request_control.time_func()) + if retry_succeeds and len(times) == 2: + return httpx.Response(200, json={"data": {"emsalKararList": [], "total": 0}}) + return httpx.Response(429, headers={"Retry-After": "9"}) + + route = respx.post(BASE_URL + "searchDocuments").mock(side_effect=response) + if retry_succeeds: + assert collect_bedesten("HGK", date(2021, 1, 1), limit=2) == [] + else: + with pytest.raises(ValueError, match="429.*retry"): + collect_bedesten("HGK", date(2021, 1, 1), limit=2) + assert route.call_count == 2 + assert times[1] - times[0] >= 9 + + +@pytest.mark.parametrize("failure", ["redirect", "mime", "base64", "private_host"]) +@respx.mock +def test_bedesten_rejects_unsafe_responses(tmp_path, monkeypatch, mock_dns, failure): + from mesa_legal_data.sources.bedesten import collect_bedesten + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_OPERATOR_CONTACT", "operator@mesa.test") + migrate() + search = fixture_json("search.json") + search["data"]["emsalKararList"] = search["data"]["emsalKararList"][:1] + search_route = respx.post(BASE_URL + "searchDocuments").respond(200, json=search) + content = fixture_json("document-1231237100.json") + if failure == "redirect": + search_route.respond(302, headers={"Location": "https://attacker.example/steal"}) + elif failure == "mime": + content["data"]["mimeType"] = "application/javascript" + elif failure == "base64": + content["data"]["content"] = "%%%invalid%%%" + elif failure == "private_host": + mock_dns("bedesten.adalet.gov.tr", "127.0.0.1") + respx.post(BASE_URL + "getDocumentContent").respond(200, json=content) + with pytest.raises((ValueError, URLFetchError)): + collect_bedesten("HGK", date(2021, 1, 1), limit=1) + conn = get_connection() + assert conn.execute("SELECT count(*) FROM artifacts").fetchone()[0] == 0 + conn.close() + + +def test_aym_esas_karar_sayisi(): + parsed = parse_decision_text( + "ANAYASA MAHKEMESİ\nEsas Sayısı : 2025/123\nKarar Sayısı : 2026/42\nKarar Tarihi: 12.02.2026" + ) + assert parsed.esas_no == "2025/123" + assert parsed.karar_no == "2026/42" + + +@respx.mock +def test_bedesten_cli(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_OPERATOR_CONTACT", "operator@mesa.test") + migrate() + respx.post(BASE_URL + "searchDocuments").respond(200, json={"data": {"emsalKararList": [], "total": 0}}) + result = CliRunner().invoke(app, ["collect", "bedesten", "--unit", "HGK", "--from", "2021-01-01", "--limit", "2"]) + assert result.exit_code == 0, result.output diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 3599d76..feb2672 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -21,3 +21,154 @@ def test_init_command(monkeypatch, tmp_path): # Running again should be safe result2 = runner.invoke(app, ["init"]) assert result2.exit_code == 0 + + +def test_source_cli_commands(monkeypatch, tmp_path): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_MESA_STAGING_DB", str(tmp_path / "mesa_staging.sqlite")) + + # Init and migrate + assert runner.invoke(app, ["init"]).exit_code == 0 + assert runner.invoke(app, ["migrate"]).exit_code == 0 + + from mesa_legal_data.catalog import ( + get_connection, + get_parser_certification, + get_source_operational_settings, + ) + + # 1. source show default + res = runner.invoke(app, ["source", "show", "mevzuat"]) + assert res.exit_code == 0 + assert "Source Operational Settings: mevzuat" in res.stdout + assert "Auto-approval Enabled : False" in res.stdout + + # 2. source auto-approval --enable + res = runner.invoke(app, ["source", "auto-approval", "mevzuat", "--enable", "--by", "test_admin"]) + assert res.exit_code == 0 + assert "Auto-approval enabled for source 'mevzuat' by test_admin." in res.stdout + + conn = get_connection() + try: + settings = get_source_operational_settings(conn, "mevzuat") + assert settings["auto_approval_enabled"] is True + assert settings["updated_by"] == "test_admin" + assert settings["enabled"] is True + assert settings["weekly_sample_count"] == 10 + finally: + conn.close() + + # 3. source auto-approval --disable + res = runner.invoke(app, ["source", "auto-approval", "mevzuat", "--disable", "--by", "test_admin2"]) + assert res.exit_code == 0 + assert "Auto-approval disabled for source 'mevzuat' by test_admin2." in res.stdout + + conn = get_connection() + try: + settings = get_source_operational_settings(conn, "mevzuat") + assert settings["auto_approval_enabled"] is False + assert settings["updated_by"] == "test_admin2" + finally: + conn.close() + + # 4. source certify-parser + res = runner.invoke( + app, + [ + "source", + "certify-parser", + "mevzuat", + "--parser", + "legislation_parser", + "--version", + "1.1.0", + "--by", + "certifier_user", + ], + ) + assert res.exit_code == 0 + assert "Parser 'legislation_parser' v1.1.0 certified for source 'mevzuat' by certifier_user." in res.stdout + + conn = get_connection() + try: + assert get_parser_certification(conn, "mevzuat", "legislation_parser", "1.1.0") is True + finally: + conn.close() + + # Check show now reflects certified parser + res = runner.invoke(app, ["source", "show", "mevzuat"]) + assert res.exit_code == 0 + assert "legislation_parser v1.1.0" in res.stdout + assert "CERTIFIED" in res.stdout + + # 5. source certify-parser --revoke + res = runner.invoke( + app, + [ + "source", + "certify-parser", + "mevzuat", + "--parser", + "legislation_parser", + "--version", + "1.1.0", + "--by", + "revoker_user", + "--revoke", + ], + ) + assert res.exit_code == 0 + assert "Parser 'legislation_parser' v1.1.0 revoked for source 'mevzuat' by revoker_user." in res.stdout + + conn = get_connection() + try: + assert get_parser_certification(conn, "mevzuat", "legislation_parser", "1.1.0") is False + finally: + conn.close() + + +def test_publish_target_cli_persists_web_defaults_and_hash(tmp_path, monkeypatch): + from mesa_legal_data.catalog import get_connection, migrate + from mesa_legal_data.publisher.engine import target_config_sha256 + from mesa_legal_data.publisher.ledger import get_mesa_target_settings + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + monkeypatch.setenv("MESA_DATA_MESA_API_KEY", "never-print-this-key") + migrate() + result = runner.invoke( + app, + [ + "publish", + "target", + "--base-url", + "http://127.0.0.1:8000", + "--tenant-id", + "tenant", + "--workspace-id", + "workspace", + "--dataset-id", + "laws", + "--agent-id", + "cli-publisher", + ], + ) + assert result.exit_code == 0, result.output + conn = get_connection() + settings = get_mesa_target_settings(conn) + conn.close() + assert settings.base_url == "http://127.0.0.1:8000" + assert (settings.tenant_id, settings.workspace_id, settings.dataset_id, settings.agent_id) == ( + "tenant", + "workspace", + "laws", + "cli-publisher", + ) + assert settings.contract_source == "configured" + assert settings.content_limit_chars == 32768 + assert settings.health_path == "/health" + assert settings.session_start_path == "/v4/sessions/start" + assert settings.publish_path == "/v4/memory/insert" + assert settings.mutation_status_path_template == "/v4/mutations/{mutation_id}" + assert settings.session_end_path_template == "/v4/sessions/{session_id}/end" + assert target_config_sha256(settings) in result.output + assert "never-print-this-key" not in result.output diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index f465952..6a2c71d 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -6,9 +6,12 @@ from mesa_legal_data.config import SettingsModel, load_settings, load_sources -def test_load_settings_defaults(): - settings = load_settings() +def test_load_settings_defaults(monkeypatch): + monkeypatch.delenv("MESA_DATA_DATA_ROOT", raising=False) + monkeypatch.delenv("MESA_DATA_MESA_STAGING_DB", raising=False) + settings = SettingsModel(_env_file=None) assert settings.data_root == "/storage/mesa-legal-data/data" + assert settings.mesa_staging_db == "/storage/mesa-legal-data/data/mesa_staging.sqlite" assert settings.environment == "development" assert settings.log_level == "INFO" assert settings.http_proxy is None diff --git a/tests/unit/test_final_closure_gate.py b/tests/unit/test_final_closure_gate.py index 8ccfa89..455622d 100644 --- a/tests/unit/test_final_closure_gate.py +++ b/tests/unit/test_final_closure_gate.py @@ -9,6 +9,7 @@ record_circuit_breaker_result, reset_circuit_breaker, ) +from mesa_legal_data.harvest.config import load_harvest_config from mesa_legal_data.harvest.models import ItemStatus from mesa_legal_data.harvest.queue import ( get_harvest_item_by_id, @@ -98,7 +99,9 @@ def fake_collect(*args, **kwargs): lambda art_id: type("PipeRes", (), {"status": "approved", "version_id": "ver-123"})(), ) - res = run_harvest_batch(db_path=db_path) + harvest_cfg = load_harvest_config() + harvest_cfg.target.minimum_free_disk_bytes = 100 + res = run_harvest_batch(harvest_cfg=harvest_cfg, db_path=db_path) assert res["processed"] == 1 assert res["succeeded"] == 1 @@ -136,7 +139,9 @@ def failing_pipeline(art_id): monkeypatch.setattr("mesa_legal_data.harvest.runner.run_pipeline_item", failing_pipeline) - res = run_harvest_batch(db_path=db_path) + harvest_cfg = load_harvest_config() + harvest_cfg.target.minimum_free_disk_bytes = 100 + res = run_harvest_batch(harvest_cfg=harvest_cfg, db_path=db_path) assert res["processed"] == 1 assert res["retry_wait"] == 1 @@ -268,7 +273,9 @@ def test_idle_harvest_run_returns_clean_zero_result(tmp_path): apply_harvest_migrations(db_path) - res = run_harvest_batch(db_path=db_path) + harvest_cfg = load_harvest_config() + harvest_cfg.target.minimum_free_disk_bytes = 100 + res = run_harvest_batch(harvest_cfg=harvest_cfg, db_path=db_path) assert res == { "processed": 0, "succeeded": 0, diff --git a/tests/unit/test_publisher_client_and_ledger.py b/tests/unit/test_publisher_client_and_ledger.py index b1bdc4d..e1eb44b 100644 --- a/tests/unit/test_publisher_client_and_ledger.py +++ b/tests/unit/test_publisher_client_and_ledger.py @@ -159,6 +159,36 @@ def assert_insert(request): assert start.called and insert.called +def test_native_v4_payload_metadata_is_flat_and_within_mesa_limit(): + client = MesaClient(MesaTargetSettings(), api_key="secret") + targets = [f"tr:legislation:law:6098:article:{n}" for n in range(1, 1000)] + chunk = SourceChunk( + chunk_id="v1:chunk:1", + document_id="doc-1", + version_id="v1", + chunk_type="article", + char_start=0, + char_end=4, + ordinal=1, + content="text", + content_hash="content-hash", + metadata={"citations": targets}, + ) + metadata = client.build_memory_insert_payload( + chunk, session_id="sess-1", idempotency_key="stable-key", finalize_revision=True + )["metadata"] + # Same measure and nesting rule as MESA's input validation. + assert len(json.dumps(metadata, sort_keys=True, default=str).encode()) <= 16 * 1024 + for value in metadata.values(): + assert not isinstance(value, dict) + if isinstance(value, list): + assert all(not isinstance(item, (dict, list)) for item in value) + kept = metadata["citations"] + assert 0 < len(kept) < len(targets) + assert kept == targets[: len(kept)] + assert chunk.metadata["citations"] == targets + + def test_native_v4_payload_rejects_chunks_larger_than_evidence_contract(): client = MesaClient(MesaTargetSettings(), api_key="secret") chunk = SourceChunk( @@ -335,3 +365,53 @@ def test_delivery_ledger_and_cross_release_dedup(db_conn): # historical delivery evidence must not be presented as current committed state. doc_status = get_document_mesa_status(db_conn, "doc-1") assert doc_status["status"] == "Update Pending" + + +@respx.mock +def test_legal_identity_only_for_known_standard_numbered_articles(monkeypatch): + import httpx + + monkeypatch.setenv("MESA_DATA_MESA_ALLOWED_HOST", "mesa-contract.test") + client = MesaClient( + MesaTargetSettings(base_url="https://mesa-contract.test", contract_source="configured"), api_key="secret" + ) + captured = [] + + def insert(request): + captured.append(json.loads(request.content)) + return httpx.Response(202, json={"status": "accepted", "mutation_id": "mut-identity"}) + + respx.post("https://mesa-contract.test/v4/memory/insert").mock(side_effect=insert) + cases = [ + ("6098", "standard", "12", "article", {"law_id": "6098", "article_id": "12"}), + ("6698", "standard", "12A", "article", {"law_id": "6698", "article_id": "12A"}), + ("2709", "standard", "12", "article", None), + ("9999", "standard", "12", "article", None), + ("6098", "temporary", "12", "article", None), + ("6098", "additional", "12", "article", None), + ("6098", "repeated", "12", "article", None), + ("6098", "standard", "12/A", "article", None), + ("6098", "standard", "12", "general", None), + ] + for law_number, article_kind, article_number, chunk_type, expected in cases: + chunk = SourceChunk( + chunk_id="v1:chunk:12", + document_id=f"tr:legislation:law:{law_number}", + version_id="v1", + chunk_type=chunk_type, + char_start=0, + char_end=4, + ordinal=1, + content="test", + content_hash="hash", + metadata={"law_number": law_number, "article_kind": article_kind, "article_number": article_number}, + ) + client.publish_source_chunk(chunk, "stable-key", session_id="sess-1", finalize_revision=True) + payload = captured[-1] + if expected is None: + assert "legal_identity" not in payload + else: + assert payload.get("legal_identity") == expected + assert payload["idempotency_key"] == "stable-key" + assert payload["revision_id"] == "v1" + assert payload["content"] == payload["evidence_span"] == "test" diff --git a/tests/unit/test_seed_collector.py b/tests/unit/test_seed_collector.py index 7d5e08b..0c3ddc7 100644 --- a/tests/unit/test_seed_collector.py +++ b/tests/unit/test_seed_collector.py @@ -1,5 +1,7 @@ import sqlite3 +import pytest +import respx from typer.testing import CliRunner from mesa_legal_data.catalog import get_db_path @@ -9,12 +11,79 @@ runner = CliRunner() +@pytest.mark.parametrize("use_local_fixture", [True, False]) +def test_changed_seed_supersedes_same_day_version(tmp_path, monkeypatch, use_local_fixture): + from mesa_legal_data.catalog import get_connection, get_document, get_version_for_artifact, migrate + from mesa_legal_data.collectors.seed import run_seed_collection + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + migrate() + document_id = "tr:legislation:law:6098" + config = tmp_path / "seed.yaml" + config.write_text( + 'seed_legislation:\n - number: "6098"\n' + f' document_id: "{document_id}"\n' + ' title: "Türk Borçlar Kanunu"\n' + ' source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6098.pdf"\n', + encoding="utf-8", + ) + fixtures = tmp_path / "fixtures" + fixtures.mkdir() + fixture = fixtures / "6098.pdf" + fixture.write_text( + "

6098 Sayılı Kanun

" + "

MADDE 1- Sözleşme tarafların karşılıklı irade açıklamalarıyla kurulur.

", + encoding="utf-8", + ) + monkeypatch.setattr( + "mesa_legal_data.sources.manual.fetch_url_stream", + lambda **kwargs: (200, {"content-type": "text/html"}, iter([fixture.read_bytes()])), + ) + kwargs = {"config_path": config, "local_fixtures_dir": fixtures if use_local_fixture else None} + first = run_seed_collection(**kwargs) + assert len(first) == 1 + conn = get_connection() + old = get_version_for_artifact(conn, first[0].artifact_id) + assert old is not None + assert get_document(conn, document_id)["current_version_id"] == old["version_id"] + old_records = conn.execute("SELECT * FROM records WHERE version_id = ?", (old["version_id"],)).fetchall() + old_bytes = (tmp_path / first[0].raw_path).read_bytes() + conn.close() + + fixture.write_text( + fixture.read_text(encoding="utf-8").replace("kurulur.", "kurulur. Yeni hüküm uygulanır."), encoding="utf-8" + ) + second = run_seed_collection(**kwargs) + assert len(second) == 1 + conn = get_connection() + new = get_version_for_artifact(conn, second[0].artifact_id) + assert new is not None + assert new["snapshot_date"] == old["snapshot_date"] + assert new["supersedes_version_id"] == old["version_id"] + assert get_document(conn, document_id)["current_version_id"] == new["version_id"] + assert get_version_for_artifact(conn, first[0].artifact_id) == old + assert conn.execute("SELECT * FROM records WHERE version_id = ?", (old["version_id"],)).fetchall() == old_records + assert (tmp_path / first[0].raw_path).read_bytes() == old_bytes + assert not conn.execute( + "SELECT code FROM validation_issues WHERE code LIKE 'VERSION_%AMBIGUITY' AND status = 'open'" + ).fetchall() + conn.close() + + repeat = run_seed_collection(**kwargs) + assert repeat[0].is_duplicate + conn = get_connection() + assert conn.execute("SELECT count(*) FROM versions WHERE document_id = ?", (document_id,)).fetchone()[0] == 2 + assert get_document(conn, document_id)["current_version_id"] == new["version_id"] + conn.close() + + def test_load_seed_config(): items = load_seed_config() - assert len(items) == 12 + assert len(items) >= 35 numbers = [i["number"] for i in items] assert "4721" in numbers assert "2709" in numbers + assert "6502" in numbers def test_collect_seed_and_report_cli(tmp_path, monkeypatch): @@ -36,29 +105,138 @@ def test_collect_seed_and_report_cli(tmp_path, monkeypatch): # Create dummy local PDF fixtures fixtures_dir = tmp_path / "fixtures" fixtures_dir.mkdir() - for num in [ - "2709", - "4721", - "6098", - "6100", - "2004", - "5237", - "5271", - "4857", - "6102", - "2577", - "1136", - "6698", - ]: + items = load_seed_config() + for item in items: + num = item["number"] pdf_file = fixtures_dir / f"{num}.pdf" pdf_file.write_bytes(f"%PDF-1.4\nFixture Content for law {num}".encode()) result = runner.invoke(app, ["collect", "seed", "--fixtures-dir", str(fixtures_dir)]) assert result.exit_code == 0, f"Error output: {result.output}" - assert "Successfully collected 12 seed legislation artifacts" in result.output + assert f"Successfully collected {len(items)} seed legislation artifacts" in result.output # Test report command rep_result = runner.invoke(app, ["report"]) assert rep_result.exit_code == 0 - assert "Total Documents: 12" in rep_result.output - assert "Total Artifacts: 12" in rep_result.output + assert f"Total Documents: {len(items)}" in rep_result.output + assert f"Total Artifacts: {len(items)}" in rep_result.output + + +def test_seed_collector_snapshot_date_and_pipeline(tmp_path, monkeypatch): + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + + # Init & Migrate + runner.invoke(app, ["init"]) + runner.invoke(app, ["migrate"]) + + db_path = get_db_path() + conn = sqlite3.connect(db_path) + conn.execute( + "INSERT OR IGNORE INTO sources (source_id, name, authority, base_url, access_mode, enabled, policy_version, config_json, created_at, updated_at) " + "VALUES ('mevzuat', 'Mevzuat', 'Gov', 'http://mevzuat.gov.tr', 'manual', 1, '1.0', '{}', '2026-08-05T00:00:00Z', '2026-08-05T00:00:00Z')" + ) + conn.commit() + conn.close() + + # Create dummy local PDF fixture for 4721 with valid text layer + fixtures_dir = tmp_path / "fixtures" + fixtures_dir.mkdir() + pdf_file = fixtures_dir / "4721.pdf" + pdf_file.write_bytes( + b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n" + b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n" + b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>\nendobj\n" + b"4 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Madde 1 - Baslik) Tj\nET\nendstream\nendobj\n" + b"5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n" + b"xref\n0 6\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n0000000241 00000 n \n0000000339 00000 n \n" + b"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n409\n%%EOF\n" + ) + + seed_yaml = tmp_path / "seed.yaml" + seed_yaml.write_text( + """ +seed_legislation: + - number: "4721" + document_id: "tr:legislation:law:4721" + title: "Türk Medeni Kanunu" + type: "law" + source_url: "https://www.mevzuat.gov.tr/MevzuatMetin/1.5.4721.pdf" +""" + ) + + from mesa_legal_data.collectors.seed import run_seed_collection + + artifacts = run_seed_collection(config_path=seed_yaml, local_fixtures_dir=fixtures_dir) + assert len(artifacts) == 1 + + art = artifacts[0] + assert art.metadata.get("snapshot_date") is not None + assert art.metadata.get("source_role") == "consolidated_text" + + # Check pipeline created a version + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT version_id, approval_status FROM versions WHERE document_id = 'tr:legislation:law:4721'") + row = cursor.fetchone() + assert row is not None, "Pipeline must run and create a version for the new artifact" + conn.close() + + # Running collect seed twice with same bytes creates no second artifact and no second version + run_seed_collection(config_path=seed_yaml, local_fixtures_dir=fixtures_dir) + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT count(*) FROM artifacts WHERE document_id = 'tr:legislation:law:4721'") + assert cursor.fetchone()[0] == 1 + cursor.execute("SELECT count(*) FROM versions WHERE document_id = 'tr:legislation:law:4721'") + assert cursor.fetchone()[0] == 1 + conn.close() + + +def test_mevzuat_run_budget_covers_the_seed_list(): + from mesa_legal_data.sources.url_fetcher import validate_source_request + + policy = validate_source_request( + source_id="mevzuat", document_family="legislation", url="https://www.mevzuat.gov.tr/MevzuatMetin/1.5.6098.pdf" + ) + assert policy.max_requests_per_run >= len(load_seed_config()) + + +@respx.mock +def test_seed_collection_keeps_one_request_budget_per_run(tmp_path, monkeypatch, capsys): + from mesa_legal_data.catalog import migrate + from mesa_legal_data.collectors.seed import run_seed_collection + from mesa_legal_data.sources import request_control + from mesa_legal_data.sources.url_fetcher import validate_source_request + + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + migrate() + urls = [f"https://www.mevzuat.gov.tr/MevzuatMetin/1.5.{n}.pdf" for n in ("6098", "4721")] + config = tmp_path / "seed.yaml" + config.write_text( + "seed_legislation:\n" + + "".join( + f' - number: "{n}"\n document_id: "tr:legislation:law:{n}"\n title: "Kanun {n}"\n' + f' source_url: "{url}"\n' + for n, url in zip(("6098", "4721"), urls, strict=True) + ), + encoding="utf-8", + ) + routes = [ + respx.get(url).respond( + status_code=200, + headers={"content-type": "text/html"}, + content=f"

MADDE 1- Kanun {i}.

".encode(), + ) + for i, url in enumerate(urls) + ] + policy = validate_source_request(source_id="mevzuat", document_family="legislation", url=urls[0]) + budget = request_control.get_run_budget("mevzuat", policy.max_requests_per_run) + for _ in range(policy.max_requests_per_run - 1): + budget.consume() + + collected = run_seed_collection(config_path=config) + + assert [a.document_id for a in collected] == ["tr:legislation:law:6098"] + assert routes[0].call_count == 1 + assert routes[1].call_count == 0 + assert "4721 Kanun 4721: failed" in capsys.readouterr().out diff --git a/tests/unit/test_source_policy.py b/tests/unit/test_source_policy.py index 0261a19..7031cb8 100644 --- a/tests/unit/test_source_policy.py +++ b/tests/unit/test_source_policy.py @@ -47,13 +47,30 @@ def test_source_policy_unknown_source(): ) -def test_source_policy_disabled_source(): - # yargitay is enabled: false in sources.yaml +def test_source_policy_disabled_source(tmp_path): + sources_yaml_path = tmp_path / "sources.yaml" + sources_yaml_path.write_text( + 'version: "1.0.0"\n' + "sources:\n" + " yargitay:\n" + " name: Disabled Yargitay\n" + " authority: Yargitay\n" + " base_url: https://karararama.yargitay.gov.tr/\n" + " access_mode: manual\n" + " enabled: false\n" + " families: [decision]\n" + " source_role: official_case_law\n" + ' policy_version: "1.0.0"\n' + " http: {user_agent: MESA-Legal-Data-Test/1.0}\n" + " allowed_content_types: [text/html]\n", + encoding="utf-8", + ) with pytest.raises(SourcePolicyError, match="SOURCE_DISABLED"): validate_source_policy( source_id="yargitay", url="https://karararama.yargitay.gov.tr/doc.html", document_family="decision", + sources_yaml_path=sources_yaml_path, ) diff --git a/uv.lock b/uv.lock index 830711a..8ce81e2 100644 --- a/uv.lock +++ b/uv.lock @@ -132,11 +132,11 @@ filecache = [ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -310,11 +310,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.32.2" +version = "4.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/98/771d8ca889f220d7d7e352addf500666368f0fd081c6080a83767580c6e4/filelock-4.2.1.tar.gz", hash = "sha256:c4c937ccac2a813cee448e662a56366cfb374c0f8702e75f295c412cc1c5d5a3", size = 650997, upload-time = "2026-10-17T16:48:14.435Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, + { url = "https://files.pythonhosted.org/packages/28/00/db89d339bc4ecdc8087fa7b8607ba22ffc27bf96286f340c9620bea428b3/filelock-4.2.1-py3-none-any.whl", hash = "sha256:7be2662e9e826b9bba28afc0f843a123280d799a2ea44bb5d35ccec67905ae44", size = 162455, upload-time = "2026-10-17T16:48:12.937Z" }, ] [[package]] @@ -1003,11 +1003,11 @@ wheels = [ [[package]] name = "pypdf" -version = "6.19.0" +version = "6.21.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/ac/63d71aaedb59acbcdef491e6ca6469165e3771c9c74358204818fd9bc5a6/pypdf-6.19.0.tar.gz", hash = "sha256:bbc43aca292369ccc6cbc8a921991ecf2538a3587ab5a116eff06c321d647155", size = 7033266, upload-time = "2026-09-16T09:32:05.946Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/2c/c43c03eaf630435f023f1dc61ec4a4a78951ad5530a62c71cc89bde307b7/pypdf-6.19.0-py3-none-any.whl", hash = "sha256:7e5d6e730e7dae87d560a2cee218b852f6498c8be61966f3cd02ead971e48d14", size = 395480, upload-time = "2026-09-16T09:32:04.087Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" }, ] [[package]]