From 5403aa0938eefdf47dd055634a5a8e55ebbde0f2 Mon Sep 17 00:00:00 2001 From: THOTH Date: Fri, 23 Oct 2026 08:33:55 +0900 Subject: [PATCH] Keep partial writes atomic, fill missing journal entries, and pick the true latest revision Journal supplement on replay - A request stored as completed whose checkpoint, notifications or operation.succeeded event were not written (for example the process stopped right after completion) now gets the missing entries filled when the same request is replayed in local mode. This applies only to 20 measured commands; the handler is not run again, entries are written once in one transaction with the actual time and a supplemented mark, and nothing is written if the chain or stored data is not in a known shape. A per-operation gate prevents a concurrent writer and a replay from filling the same tail. Hosted review mode is unchanged. No half-written records - memory/validate, memory/changeSet/propose and memory/retire/propose write their two MEMORY control records in one short transaction, so a failure on the second write leaves no row. - R2 closed-loop failures write the sandbox rejection and the research-test hold together, after the sandbox work has finished. A final non-retried failure writes completion, admission and recovery candidate in one outer transaction. Retry, plain and success paths keep their boundaries, and no transaction is held while the sandbox runs or across an await. SandboxService.accepting_observation gains an opt-in defer_rejection keyword (default unchanged). Clear message when only the reference step fails - When research was committed but the later reference step failed, the error now says, in Korean and English, that the research (in v2: its intermediate records) was saved, gives the reason, and asks the user to check the saved result before sending the same question again. data.research_saved is true. The notice appears only after the research commit is proven readable for this project, thread and request; otherwise the previous error is returned. Error codes, reason_code and the recorded exception type are unchanged. Current revision under clock changes - Stores chose the current revision by created_at, so a newer revision written while the system clock had moved back lost to an older one (18 of 19 checked tables). All affected writers append rows, so the current revision is now the last stored one: plan executions and outcome series by their revision number then rowid, control records by version then rowid (read and latest list), and the other tables by rowid. Schema, writers, canonical heads and history lists are unchanged; the thread checkpoint list keeps its display order while pause/resume guards use the real latest checkpoint. Two control-record revisions with the same version can still be written concurrently; the choice is now deterministic. Verification: full suite on the exact tree - Python 5503 tests (0 failures, 0 errors, 2 skipped), web tests, lint, typecheck and build, ruff, architecture and module budget checks, and the README quick-start smoke test all pass. --- SOURCE_MANIFEST.json | 242 +++++-- apps/web/src/api/rpcClient.test.ts | 40 ++ config/module-responsibility-budget.json | 2 +- docs/KNOWN_LIMITATIONS.ko.md | 21 +- docs/KNOWN_LIMITATIONS.md | 27 +- src/thoth/adapters/storage/action.py | 18 +- src/thoth/adapters/storage/control_record.py | 15 +- .../adapters/storage/criterion_contract.py | 6 +- src/thoth/adapters/storage/decision_object.py | 16 +- src/thoth/adapters/storage/events.py | 39 +- src/thoth/adapters/storage/evidence_graph.py | 5 +- src/thoth/adapters/storage/execution.py | 20 +- src/thoth/adapters/storage/hypothesis.py | 17 +- src/thoth/adapters/storage/outcome.py | 20 +- src/thoth/adapters/storage/thread_runtime.py | 4 +- src/thoth/adapters/storage/workspace_owner.py | 5 + src/thoth/application/commands/memory_full.py | 126 ++-- .../application/commands/memory_records.py | 24 + src/thoth/application/commands/reference.py | 20 +- src/thoth/application/commands/threads.py | 5 +- .../application/services/r2_closed_loop.py | 190 ++--- .../services/r2_failure_publication.py | 56 ++ .../services/r2_recovery_records.py | 51 +- .../application/services/reference_failure.py | 112 +++ .../services/research_attempt_runner.py | 5 +- .../services/research_termination.py | 8 +- .../application/services/sandbox_service.py | 5 +- src/thoth/apps/reference_composition.py | 1 + src/thoth/apps/resource_scope_composition.py | 3 + src/thoth/apps/runtime_entry.py | 20 +- src/thoth/ports/journal.py | 17 +- src/thoth/protocol/bus.py | 66 +- src/thoth/protocol/journal_completion.py | 242 +++++++ src/thoth/protocol/journal_completion_gate.py | 34 + .../protocol/journal_completion_methods.py | 213 ++++++ src/thoth/protocol/operation_scope.py | 16 + .../operation_result_digest_vectors.json | 44 ++ tests/fixtures/r2_closed_loop_golden.json | 55 +- tests/fixtures/reference_service_golden.json | 5 +- tests/fixtures/supplementable_methods.json | 341 +++++++++ .../integration/journal_completion_helpers.py | 81 +++ tests/integration/p_r2_failure_helpers.py | 83 +++ .../test_journal_completion_concurrency.py | 127 ++++ .../test_journal_completion_faults.py | 143 ++++ .../test_journal_completion_process.py | 145 ++++ .../test_journal_completion_replay.py | 173 +++++ ..._m3_hypothesis_first_write_reachability.py | 12 +- ...test_m3_substantive_appraisal_atomicity.py | 12 +- ...4_memory_control_multiwrite_observation.py | 44 +- ...st_operation_journal_failure_boundaries.py | 26 +- tests/integration/test_p_memory_atomicity.py | 150 ++++ .../test_p_r2_failure_atomicity.py | 231 ++++++ .../test_p_reference_failure_notice.py | 256 +++++++ .../test_storage_clock_currentness.py | 662 ++++++++++++++++++ tests/unit/test_journal_completion.py | 319 +++++++++ tests/unit/test_journal_completion_gate.py | 92 +++ 56 files changed, 4313 insertions(+), 399 deletions(-) create mode 100644 apps/web/src/api/rpcClient.test.ts create mode 100644 src/thoth/application/commands/memory_records.py create mode 100644 src/thoth/application/services/r2_failure_publication.py create mode 100644 src/thoth/application/services/reference_failure.py create mode 100644 src/thoth/protocol/journal_completion.py create mode 100644 src/thoth/protocol/journal_completion_gate.py create mode 100644 src/thoth/protocol/journal_completion_methods.py create mode 100644 src/thoth/protocol/operation_scope.py create mode 100644 tests/fixtures/operation_result_digest_vectors.json create mode 100644 tests/fixtures/supplementable_methods.json create mode 100644 tests/integration/journal_completion_helpers.py create mode 100644 tests/integration/p_r2_failure_helpers.py create mode 100644 tests/integration/test_journal_completion_concurrency.py create mode 100644 tests/integration/test_journal_completion_faults.py create mode 100644 tests/integration/test_journal_completion_process.py create mode 100644 tests/integration/test_journal_completion_replay.py create mode 100644 tests/integration/test_p_memory_atomicity.py create mode 100644 tests/integration/test_p_r2_failure_atomicity.py create mode 100644 tests/integration/test_p_reference_failure_notice.py create mode 100644 tests/integration/test_storage_clock_currentness.py create mode 100644 tests/unit/test_journal_completion.py create mode 100644 tests/unit/test_journal_completion_gate.py diff --git a/SOURCE_MANIFEST.json b/SOURCE_MANIFEST.json index 5c8f14a..32adc81 100644 --- a/SOURCE_MANIFEST.json +++ b/SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schema_version": "thoth-source-preview-manifest-v1", - "created_at": "2026-10-19T13:28:27.492646+00:00", + "created_at": "2026-10-22T18:37:46.948775+00:00", "license": "MIT", "git_history_included": false, "full_suite_verified": false, @@ -411,6 +411,11 @@ "sha256": "9edd2a0cccfbd903f70a85787f05274b402dee7c6e6b4e454078ab8dfcfe97d3", "bytes": 1709 }, + { + "path": "apps/web/src/api/rpcClient.test.ts", + "sha256": "aad467a95444ab712ca5cede7034d6acb4ac0d4c94f58c77355a7896123d1ebf", + "bytes": 2163 + }, { "path": "apps/web/src/api/rpcClient.ts", "sha256": "92a77302b04086eca3e3c422248b81017586cbaaca4054370149043b16407c69", @@ -1878,7 +1883,7 @@ }, { "path": "config/module-responsibility-budget.json", - "sha256": "2c8eee608d261d612e04010f1e3e18bd1df3549a91b91d323beee60316924f58", + "sha256": "6992a0c3aa7cc3822aa5c7eaa461e04961ba89809ae22797c914676469e5655a", "bytes": 8580 }, { @@ -1948,13 +1953,13 @@ }, { "path": "docs/KNOWN_LIMITATIONS.ko.md", - "sha256": "777febcf50b13b2878658fe826e34c0230d3d9930efbf9cea72897facfcefe60", - "bytes": 8864 + "sha256": "ce5cc699dc080560681b09f0f7790365e13d57922c0f9d4ba117152b6d297e27", + "bytes": 10028 }, { "path": "docs/KNOWN_LIMITATIONS.md", - "sha256": "f294c9aaaefcca87c0d61f9704d0bbf3a9a97568bcf00570467eec74128ce113", - "bytes": 7626 + "sha256": "6d7bbcb9c5b7e392c19b8d44d2dc7bccb03741cae9d524983f856a6efeb690d4", + "bytes": 8541 }, { "path": "docs/KNOWN_TEST_FAILURES.json", @@ -3923,8 +3928,8 @@ }, { "path": "src/thoth/adapters/storage/action.py", - "sha256": "26f9db308116e146d36c5cb173c6e49bdce714e37f77a98ecfe2c41aa35ad1f9", - "bytes": 16247 + "sha256": "bc81daab38f70a8444d20b1172b701b1b2bf8614759389235710e67c6d5cafec", + "bytes": 15954 }, { "path": "src/thoth/adapters/storage/action_schema.py", @@ -3993,8 +3998,8 @@ }, { "path": "src/thoth/adapters/storage/control_record.py", - "sha256": "ea1b264b5a0fe73086e756f36482cff17e2f5753de32af51a69f819cfc11e157", - "bytes": 4345 + "sha256": "1683f971c489ea923de3f46a5799336e05a96136291756175d1e595ddc5797da", + "bytes": 4586 }, { "path": "src/thoth/adapters/storage/conversation.py", @@ -4003,8 +4008,8 @@ }, { "path": "src/thoth/adapters/storage/criterion_contract.py", - "sha256": "523bc48f72511909c0ca29e65bcecc217a9b00b911444fe45f04ef9e42f2c2ce", - "bytes": 17105 + "sha256": "000584b6882d1d688f3072fade00407a10d6735fc8cb10584c1d3b389ecb1fed", + "bytes": 17000 }, { "path": "src/thoth/adapters/storage/criterion_schema.py", @@ -4013,8 +4018,8 @@ }, { "path": "src/thoth/adapters/storage/decision_object.py", - "sha256": "e1509f97eaab1b95bf928c4e6b41faa7917e6a01dc792dcf7b7aa5810d198b7d", - "bytes": 16633 + "sha256": "58ccbf0e263e6b41b13fc57bb6a742e964a74193190a83bd1096f1e6e67bf77d", + "bytes": 16718 }, { "path": "src/thoth/adapters/storage/decision_object_schema.py", @@ -4038,13 +4043,13 @@ }, { "path": "src/thoth/adapters/storage/events.py", - "sha256": "0ed076d668ab5abe9d912894e01401941e92ca44059ab518efd9159d0376b3d1", - "bytes": 4340 + "sha256": "80ec394619aae2688f935810058b445aa301d5d7134535d6352cc8b7ab606af2", + "bytes": 5757 }, { "path": "src/thoth/adapters/storage/evidence_graph.py", - "sha256": "292302b0e7572426659d5dc5ea73025e40b310c0079e13ed7e03fd0bce1c40cc", - "bytes": 19720 + "sha256": "bc721ea2d9b4a4b2202dd036adb0d3cf2ce50609b63d2a974d6eb7dad66944c5", + "bytes": 19611 }, { "path": "src/thoth/adapters/storage/evidence_schema.py", @@ -4053,8 +4058,8 @@ }, { "path": "src/thoth/adapters/storage/execution.py", - "sha256": "e1ce36bc08f88942880ec9c25f852aea1bdcfd30ab616696c90299dc2e90eb85", - "bytes": 10260 + "sha256": "d9a78985468c0e40f4821d284249928651f94f49cfeb03efc33ce897c679aa41", + "bytes": 10381 }, { "path": "src/thoth/adapters/storage/execution_schema.py", @@ -4093,8 +4098,8 @@ }, { "path": "src/thoth/adapters/storage/hypothesis.py", - "sha256": "3be522d8cd6ed0705c7055f9e333e8d59c9ec605c6cdbc7c07cde3996d060a68", - "bytes": 19201 + "sha256": "01f25bafe4267fbd190739db9abff21aeeeff2ad2955c5a99fa7e80bfed6fcf7", + "bytes": 18890 }, { "path": "src/thoth/adapters/storage/hypothesis_schema.py", @@ -4138,8 +4143,8 @@ }, { "path": "src/thoth/adapters/storage/outcome.py", - "sha256": "cdd121cfb79d8d4dd7559d889b18e5962a07e12375e895cd4a5111258a4b5b9d", - "bytes": 15233 + "sha256": "bf730a9c6e46203dc07aa21e34b0e419564356ee22c5a101b036b48860e2dcd0", + "bytes": 15643 }, { "path": "src/thoth/adapters/storage/outcome_schema.py", @@ -4223,8 +4228,8 @@ }, { "path": "src/thoth/adapters/storage/thread_runtime.py", - "sha256": "c1e2f9c13bdcc90af08369e68f39f5e9b945036aaf42ff6637ed4c4a3c933f74", - "bytes": 7879 + "sha256": "0153010a13fb79114d58c8f2fe30043b2eea4a7b58e9ba116061b5bccaf35de6", + "bytes": 7871 }, { "path": "src/thoth/adapters/storage/threads.py", @@ -4238,8 +4243,8 @@ }, { "path": "src/thoth/adapters/storage/workspace_owner.py", - "sha256": "ec0f5d99b5e71ba00fc16f988ace72779788cdaa0a9f1ac011646ebf5f213584", - "bytes": 6935 + "sha256": "7138a59c44a2c25e9d5a79df5112caff82ebb286ab61a6c7c1c36f250ea78989", + "bytes": 7097 }, { "path": "src/thoth/adapters/storage/workspace_setup.py", @@ -4468,8 +4473,8 @@ }, { "path": "src/thoth/application/commands/memory_full.py", - "sha256": "5ea4c7510c781623f59a5e8ea64688470c33543913d448613511f9091117aa97", - "bytes": 22235 + "sha256": "ac7bf6c342d1d293684a0026e69e68ff8c18e9ff30b3e56cda45a17e17a30881", + "bytes": 22173 }, { "path": "src/thoth/application/commands/memory_inputs.py", @@ -4481,6 +4486,11 @@ "sha256": "d893dd4b581e95e6eaa4757fbec7ac9a3333238b9080d137a6fa22c056786d15", "bytes": 1815 }, + { + "path": "src/thoth/application/commands/memory_records.py", + "sha256": "53bea1a5b4742c4a76e576e2a351dcb1082aa7d25b718fd54bdf841cbbec0633", + "bytes": 695 + }, { "path": "src/thoth/application/commands/memory_revisions.py", "sha256": "0a79894a19107114b6669f342b178a0fe62157a7a02434307a41a4fd27ef88c9", @@ -4563,8 +4573,8 @@ }, { "path": "src/thoth/application/commands/reference.py", - "sha256": "469d3b8fe4f47a28fe040fad4619f1b27650cdf14acfbdda07a18caa9f6ac37b", - "bytes": 7351 + "sha256": "0dc5aca4c53c8c75879c95fd6f39fc5d247f03f669dbae98d56d7db8a8b260b1", + "bytes": 7740 }, { "path": "src/thoth/application/commands/research_followup.py", @@ -4668,8 +4678,8 @@ }, { "path": "src/thoth/application/commands/threads.py", - "sha256": "4335b93071fcba7d318304e7e1dbd239c0a573f1cbe8343e57af0561fa06b6a1", - "bytes": 23304 + "sha256": "329fcd5eda0f860baeb4f341f64e09797c0ece261ccb5fbdc4e62c08a7495661", + "bytes": 23439 }, { "path": "src/thoth/application/commands/trace_closure.py", @@ -5423,8 +5433,8 @@ }, { "path": "src/thoth/application/services/r2_closed_loop.py", - "sha256": "0970aa27c14e0eb4f905f883120500f00c599d73f9cbe0944ca2f97726c1f74f", - "bytes": 24940 + "sha256": "4ef8fbb555f3e2d1864902f668452b9e410161f9324c63ba82df8f314a97aa36", + "bytes": 25475 }, { "path": "src/thoth/application/services/r2_context_preflight.py", @@ -5436,6 +5446,11 @@ "sha256": "2cb3a97cc78385c42e8d8b88ce87497c5df979f6758a0d8577b1d038fec743b7", "bytes": 7373 }, + { + "path": "src/thoth/application/services/r2_failure_publication.py", + "sha256": "625fe6a36fbb42eaa599a8622786239aabb7939c6d5adab33d7a030c914122d4", + "bytes": 2072 + }, { "path": "src/thoth/application/services/r2_outcome_projection.py", "sha256": "fec137ddbb3297dce6ab21e24ccfcd7d18a60dead1710638881d54b6d292bd59", @@ -5443,8 +5458,8 @@ }, { "path": "src/thoth/application/services/r2_recovery_records.py", - "sha256": "912fcbbcf07a91d22620039eb49f8ecf7da7b40330924f36432a09027c978bfb", - "bytes": 8894 + "sha256": "f73b0035ba36fc82b91f645eba2b57980665339bd4ecf3cdf9a8d2f62bc74b23", + "bytes": 10222 }, { "path": "src/thoth/application/services/r2_sandbox_compiler.py", @@ -5471,6 +5486,11 @@ "sha256": "aec4728dab5a7ae2a5ca20bcf0fc8264f31fa2629bfc63ebf9b40b59d1bdb188", "bytes": 7823 }, + { + "path": "src/thoth/application/services/reference_failure.py", + "sha256": "1bd6711e072fc06027a80d014b5dc4454556ca9e60bdaad0907a504f312e425f", + "bytes": 4598 + }, { "path": "src/thoth/application/services/reference_grounding.py", "sha256": "c39afc2d0e5d37ff204870a5886ad54729a95ac36cad60b9641ea6e1598cf525", @@ -5523,8 +5543,8 @@ }, { "path": "src/thoth/application/services/research_attempt_runner.py", - "sha256": "868fab123ff6482f97281f1d853e8208324161a7f21227da76a3b4ff1e333acf", - "bytes": 18844 + "sha256": "5e6499afabe6ea7d2d10f80123e8070f55eee4a265c02d2c06901e0d329dc088", + "bytes": 18958 }, { "path": "src/thoth/application/services/research_basis_capture.py", @@ -5743,8 +5763,8 @@ }, { "path": "src/thoth/application/services/research_termination.py", - "sha256": "25c78dca2407f3b5a7514f5bd22291f7f5b5ee304b408a4f33bcb232b576f470", - "bytes": 5501 + "sha256": "8c4e550c2d90d486a1f3c548ceb300ae72364cb89b82d8a9d2703cc222f2d224", + "bytes": 5824 }, { "path": "src/thoth/application/services/research_thread_read_snapshot.py", @@ -5858,8 +5878,8 @@ }, { "path": "src/thoth/application/services/sandbox_service.py", - "sha256": "669690aa50057b4bb208b360381f12f84bf7356f25ca59cb50382b75bd11d3bf", - "bytes": 25937 + "sha256": "0b053c9acbcdbdd2344f6c29035930ad63d0919f861008aad56b40eca95ce25c", + "bytes": 26030 }, { "path": "src/thoth/application/services/scoped_artifacts.py", @@ -6168,8 +6188,8 @@ }, { "path": "src/thoth/apps/reference_composition.py", - "sha256": "53e72e2f0a9ab0753fd79720fc94f28e1fb364bfb2e71f1bf29d4356e811ab5e", - "bytes": 1652 + "sha256": "5b4f185fb11a0f6140349c517163cf5876666843b73b1635c90c2551acdeebef", + "bytes": 1668 }, { "path": "src/thoth/apps/research_entry_composition.py", @@ -6188,8 +6208,8 @@ }, { "path": "src/thoth/apps/resource_scope_composition.py", - "sha256": "f29ad10251d86251ac70f5c6330ef08db1fefa631b3ad66c7e8ed41d0bdfca5a", - "bytes": 7204 + "sha256": "478622520453bed77125dcd5da84133618a442550ac87cdc9fc80bc9b3a7f92a", + "bytes": 7372 }, { "path": "src/thoth/apps/restore_profiles.py", @@ -6208,8 +6228,8 @@ }, { "path": "src/thoth/apps/runtime_entry.py", - "sha256": "b9bc5d885152e0fe4bc084678b8f17810facb37313a4c46651f1ce481413b50d", - "bytes": 6005 + "sha256": "6a81df4e30b133962fa1051f6effb2ceef2e68a33fe285c418189c393e036f6d", + "bytes": 6763 }, { "path": "src/thoth/apps/runtime_registration.py", @@ -7048,8 +7068,8 @@ }, { "path": "src/thoth/ports/journal.py", - "sha256": "6eabc143facec100dc053fe98f57b024e31697ae9962afee7e1e38b0ba32f692", - "bytes": 694 + "sha256": "87985bde73c736d1be3f3d06ba30fe33272fe28f9ff12b5019d7b0381a779fbb", + "bytes": 1368 }, { "path": "src/thoth/ports/ledger.py", @@ -7223,14 +7243,29 @@ }, { "path": "src/thoth/protocol/bus.py", - "sha256": "5a6f0012d36f4e719080566385916ce7441977baaaaaa6127f970b8b2f2d06a7", - "bytes": 31488 + "sha256": "95e281e6ddbebbe87abde04fc37b437785d8511cc9138bf2551bd156c674624b", + "bytes": 31656 }, { "path": "src/thoth/protocol/deferred.py", "sha256": "c4ec6f0d306c20b48ded1a7ebdee4d69f1d0e91d294792f39d0065472b40fdc1", "bytes": 927 }, + { + "path": "src/thoth/protocol/journal_completion.py", + "sha256": "f0d2a939b376d8f0998feaff4d52102fbafe0a9fe51da622e19242402315fbea", + "bytes": 9840 + }, + { + "path": "src/thoth/protocol/journal_completion_gate.py", + "sha256": "e6bbd755fcb053e7d25f026f21e115e974066fc2bd6ebc5daaebeb3b2c333468", + "bytes": 917 + }, + { + "path": "src/thoth/protocol/journal_completion_methods.py", + "sha256": "4a19219b6baf0c179a0ae14ddd26502ff20399b31f219ca9dd10a6e5298d64b1", + "bytes": 4365 + }, { "path": "src/thoth/protocol/jsonrpc.py", "sha256": "8da5f77e17afb3e73e3355de50dfa3a857366472a7080d380d16daec88d2b67c", @@ -7261,6 +7296,11 @@ "sha256": "8f6da0b375e71715fe8ab4b959b8c41b9be1f7b826075ff0bc937dad4f0b8c73", "bytes": 8187 }, + { + "path": "src/thoth/protocol/operation_scope.py", + "sha256": "87f430719f2bdef2e103edaa3c348937b71c9cb9cc36f96e2495724a4abe175d", + "bytes": 470 + }, { "path": "src/thoth/protocol/read_queries.py", "sha256": "17a7460e4c4be9e9b07c2cda5240346e758fa6bc1410e259c02596ed6ea81333", @@ -7881,6 +7921,11 @@ "sha256": "7f62d3b3ffc64c620be19ca6b0786d9a775c1c3be198727c281e7cfad0e67366", "bytes": 1242 }, + { + "path": "tests/fixtures/operation_result_digest_vectors.json", + "sha256": "812d4554d29fb41871c1ba9b49a577881112001ce0eedc51bb7bbe4851869838", + "bytes": 1022 + }, { "path": "tests/fixtures/outcome_response_golden.json", "sha256": "8994fb66738ff2926e0a17586b73f1156fe433d5d570c4671b5e56a6b3ed55fb", @@ -7893,8 +7938,8 @@ }, { "path": "tests/fixtures/r2_closed_loop_golden.json", - "sha256": "4afc05462d53e7324146f0bc6ae38c5903792c5cf9ed06b16774df3678d4be63", - "bytes": 653428 + "sha256": "594820e171b9221fc34c36b66ce414f75df601ac64968e7af9bb56bd907caaa1", + "bytes": 653944 }, { "path": "tests/fixtures/recursive_improvement_golden.json", @@ -7903,8 +7948,8 @@ }, { "path": "tests/fixtures/reference_service_golden.json", - "sha256": "9f9e0ab91a4150a6c526ed270fcb84e633fb4fb822cde1b4cc88428d1f7909e2", - "bytes": 1126800 + "sha256": "9f29e71a60114ba4716835467842b86311d314347694036adf3070496b63af31", + "bytes": 1127566 }, { "path": "tests/fixtures/resource_scope_records_golden.json", @@ -7966,6 +8011,11 @@ "sha256": "46cbc6121770020d167dc3c24bfa4cfd0716c67e94a1b1b73745985303314a93", "bytes": 3758 }, + { + "path": "tests/fixtures/supplementable_methods.json", + "sha256": "61e0c7811e4999eb1e79dc3f639ea723c73becc5c1fe4033eacf4d71146f6cff", + "bytes": 7309 + }, { "path": "tests/fixtures/trace_csv/README.txt", "sha256": "164fbf3abc1fe39f18ef3a5ab1645da3030dcebf6ae6a137515ba219340e60e2", @@ -8051,11 +8101,21 @@ "sha256": "32357695636c28b0bef0b8d3c0d9e861dee2ab516b62192bd8c0f4ec8e40d804", "bytes": 31117 }, + { + "path": "tests/integration/journal_completion_helpers.py", + "sha256": "51bc70fd4bdd2d2a8bd357bda4611231d79bb9a9a91aa9857dd880f0de8c82bc", + "bytes": 2655 + }, { "path": "tests/integration/migration_head_helpers.py", "sha256": "558317d21741346f81c115e2d2c60ea461d0831b60319593509756f470db7998", "bytes": 698 }, + { + "path": "tests/integration/p_r2_failure_helpers.py", + "sha256": "0e2c5aa23834785508af97a8638b8619e80d10b641812ced3fcb64980f09a7d9", + "bytes": 2902 + }, { "path": "tests/integration/paired_evaluation_helpers.py", "sha256": "82966f164bd44062dc3655b5db0d7bfcb47e428c93fe9b34a11ef5ffa1ca8f4d", @@ -8731,6 +8791,26 @@ "sha256": "5e64d55d216651b3c1eb2669c9bb4eb04b881cd15e12dcb05d288af1427f98f0", "bytes": 7761 }, + { + "path": "tests/integration/test_journal_completion_concurrency.py", + "sha256": "fa144f687f36b5134ed6d31fe723af56173c9a7504eba9fa5612dec5043ae0ef", + "bytes": 5320 + }, + { + "path": "tests/integration/test_journal_completion_faults.py", + "sha256": "6e21c08dbdc31a505437a8597331344aa3e66235fdf964db9b0cceaa1fe9c1e0", + "bytes": 6324 + }, + { + "path": "tests/integration/test_journal_completion_process.py", + "sha256": "36d4d0c86b33b16b0a7821e0dcbae49f19ee33fe0c776bc81b7e071075c8c470", + "bytes": 6171 + }, + { + "path": "tests/integration/test_journal_completion_replay.py", + "sha256": "a797e1e289a35697e872c86a56be0943c41e3711ae139663273d2debcdd692be", + "bytes": 6877 + }, { "path": "tests/integration/test_judgment_records.py", "sha256": "7778ae97e6d23da3efd9d5e81658c2c63d05e2fa4fe02949fed93e3e4fc2ccbb", @@ -8828,8 +8908,8 @@ }, { "path": "tests/integration/test_m3_hypothesis_first_write_reachability.py", - "sha256": "273625a7ce30feaa6598b056d74d40a641271d3239db9edafe73c92971e63733", - "bytes": 18476 + "sha256": "edcbd23477dc1be0e1d4a66ac5d4d064d246e31ec8104e18427b19eade71264e", + "bytes": 18650 }, { "path": "tests/integration/test_m3_hypothesis_mutation_atomicity.py", @@ -8873,13 +8953,13 @@ }, { "path": "tests/integration/test_m3_substantive_appraisal_atomicity.py", - "sha256": "5810761fbcebd2723ec129e5084b2e4fa22f113bbfe69bdbad55a303bd8545f6", - "bytes": 17895 + "sha256": "684854ad58dfb159f5bfe2fca75bc4ea8ed3c9becec9a42f99768728e1e5e2e7", + "bytes": 18069 }, { "path": "tests/integration/test_m4_memory_control_multiwrite_observation.py", - "sha256": "49663656a8527b1f8c10311ef1edc731be6bee31e8af56293c4af46b649883ef", - "bytes": 18382 + "sha256": "df0eb3fb72777694a1dcbdd11750341e35915de8150b66649c00beb176376ba3", + "bytes": 17799 }, { "path": "tests/integration/test_m5_exposure_service_atomicity.py", @@ -9048,8 +9128,8 @@ }, { "path": "tests/integration/test_operation_journal_failure_boundaries.py", - "sha256": "7b269e5c1382f64823122be46004ea6e0679725d3ba6b6d9644523596e37a927", - "bytes": 13419 + "sha256": "bf2b8ff1039d1455e4702ec92115b4785ee2a54239f9fefe3d7fa51715459d7b", + "bytes": 14433 }, { "path": "tests/integration/test_operation_owner_atomicity.py", @@ -9091,6 +9171,21 @@ "sha256": "ba727b080bd12284ca1e80604154a79e8d53fdd310a0a67cc673536d73146785", "bytes": 7738 }, + { + "path": "tests/integration/test_p_memory_atomicity.py", + "sha256": "a3e4b3f143fa1c3f3a0e5b685f0ff998acb05ae5c15145f1350ca465efdc0467", + "bytes": 5637 + }, + { + "path": "tests/integration/test_p_r2_failure_atomicity.py", + "sha256": "cbf8e3a7140850802a253d72f131de415716487e047dadd6cd850a535f3de231", + "bytes": 10405 + }, + { + "path": "tests/integration/test_p_reference_failure_notice.py", + "sha256": "f8c40869a0331027c8375af1c9d48bbb6a79af66baabcb50836bcc73cafc4cfa", + "bytes": 11541 + }, { "path": "tests/integration/test_paired_evaluation_execution.py", "sha256": "d68853afb5cfe88014dd7dc3cfa446b5580a9132da5e21ab7d40e692ed87db5c", @@ -9756,6 +9851,11 @@ "sha256": "1620bfa1c34d9c17cc5ed1477893596c720b097df597b99afa70f2e4ac919ba6", "bytes": 18126 }, + { + "path": "tests/integration/test_storage_clock_currentness.py", + "sha256": "5fda6124d80d745b9fce8bd8095dc191a3b9d9a5931dd02cda94cfd52e05600f", + "bytes": 23428 + }, { "path": "tests/integration/test_storage_migration_ddl_contract.py", "sha256": "55e246ca9efa5312baa9c7b39dc22361a13efd55680486a75cf62dbdca171c21", @@ -11146,6 +11246,16 @@ "sha256": "88e55392895bb63a1da1ad7c26abbee88778d813194bd29779ecb51dea18020f", "bytes": 10661 }, + { + "path": "tests/unit/test_journal_completion.py", + "sha256": "37896a225b6fc37e3d80fa962f895ad9848840f7f0377f3cf763d6a5433fdc1f", + "bytes": 12376 + }, + { + "path": "tests/unit/test_journal_completion_gate.py", + "sha256": "471f7cfebed4b6694007c79e90f2c565875bee8f537ad650046d480b9d6e7928", + "bytes": 3179 + }, { "path": "tests/unit/test_lesson_recall_cases.py", "sha256": "a20593ab7789905988c265ca75f5297575d57e39f3d5c81ee8980bb110dd1ddb", diff --git a/apps/web/src/api/rpcClient.test.ts b/apps/web/src/api/rpcClient.test.ts new file mode 100644 index 0000000..2aee9e2 --- /dev/null +++ b/apps/web/src/api/rpcClient.test.ts @@ -0,0 +1,40 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { rpc, rpcAsync } from "./rpcClient"; + +afterEach(() => vi.unstubAllGlobals()); + +const reason = "REFERENCE_INQUIRY_NOT_FOUND"; +const notice = `연구의 중간 기록은 저장됐습니다. 기준(reference) 단계만 실패했습니다 (사유: ${reason}). ` + + "결과는 기록에서 볼 수 있습니다. 같은 질문을 새로 보내면 연구가 다시 실행될 수 있으니 먼저 저장된 결과를 확인하세요. / " + + `Intermediate research records were saved. Only the reference step failed (reason: ${reason}). ` + + "You can read the result in the history. Sending the same question again as a new request may run the research again, so check the saved result first."; + +it("preserves the bilingual reference notice and saved-record metadata from an RPC error", async () => { + const data = { reason_code: reason, research_saved: true }; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ + jsonrpc: "2.0", id: "notice", error: { code: -32030, message: notice, data }, + }))); + await expect(rpc("thread/input", { project_id: "p" }, "reference-notice")).rejects.toMatchObject({ + name: "RpcError", code: -32030, message: `${notice} [-32030]`, details: data, + }); +}); + +it("shows the saved asynchronous operation's reference notice without replacing it", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(Response.json({ + jsonrpc: "2.0", id: "started", + result: { operation_id: "op:notice", state: "RUNNING", value: { status: "ACCEPTED_RUNNING" } }, + })) + .mockResolvedValueOnce(Response.json({ + operation_id: "op:notice", state: "FAILED", result: null, + error: { code: -32603, message: notice, data: { reason_code: reason, research_saved: true } }, + })); + vi.stubGlobal("fetch", fetch); + await expect(rpcAsync("thread/input", { project_id: "p", contract_version: 2 }, "async-notice", { + pollMs: 0, + })).rejects.toMatchObject({ message: notice }); + expect(fetch).toHaveBeenCalledTimes(2); + expect(fetch).toHaveBeenLastCalledWith("/operations/op%3Anotice?project_id=p", { + credentials: "same-origin", + }); +}); diff --git a/config/module-responsibility-budget.json b/config/module-responsibility-budget.json index 4a6d579..4ffacbf 100644 --- a/config/module-responsibility-budget.json +++ b/config/module-responsibility-budget.json @@ -123,7 +123,7 @@ "policy": "over the 600-line limit when it was introduced; do not grow it; lower the baseline when it is split" }, "src/thoth/application/commands/memory_full.py": { - "baseline_lines": 519, + "baseline_lines": 513, "top_level_symbols": [ "MemoryFullHandlers" ], diff --git a/docs/KNOWN_LIMITATIONS.ko.md b/docs/KNOWN_LIMITATIONS.ko.md index 7c4f2c7..5537ad2 100644 --- a/docs/KNOWN_LIMITATIONS.ko.md +++ b/docs/KNOWN_LIMITATIONS.ko.md @@ -38,11 +38,24 @@ handler를 다시 실행하거나 과거 요청을 자동으로 훑어 닫지는 있고, 새로 읽은 개정 번호로 보내면 같은 변경이 한 번 더 적용될 수도 있습니다. 이 복구가 외부 효과의 정확히 한 번 실행을 보장하는 것은 아닙니다. -## 끝난 요청의 후속 활동 기록이 빠질 수 있습니다 +## 같은 요청 재전송으로 빠진 성공 일지를 보충할 수 있습니다 -요청이 끝나면 결과를 먼저 저장하고, 활동 기록의 checkpoint·알림·"성공" 항목은 그 뒤에 저장합니다. -둘 사이에 저장소 오류나 종료가 겹치면 결과와 연결 정보는 그대로이고 같은 요청은 그 결과를 -돌려주지만, 후속 항목은 빠질 수 있고 나중에 채워지지 않습니다. 요청을 다시 할 필요는 없습니다. +로컬 모드에서는 실측한 20개 명령과 지원되는 저장 결과 형식에 한해, 같은 멱등 key를 다시 보내면 +빠진 checkpoint·알림·성공 기록을 채울 수 있습니다. 저장된 결과와 연결 정보는 그대로이며 +handler·모델·그 밖의 효과를 다시 실행하지 않습니다. 기록이 다 채워지면 재전송해도 추가하지 않습니다. + +새 항목의 시각은 실제 보충 시각이며, `supplemented: true`와 원 완료 시각을 담은 +`supplemented_for`가 붙습니다. 기존 항목은 고치지 않습니다. 보충 중 중단되면 그 거래의 새 기록은 +함께 취소되고 다음 같은 요청에서 다시 시도할 수 있습니다. + +심사용 모드, 실패·진행 중 요청, 연구 게시와 복원 게시에는 적용하지 않습니다. 다른 명령과 +확인하지 못한 옛 저장 형식도 보류합니다. 일지 형식이 불명확하거나 서로 충돌하는 경우, 시계 역행 +등으로 이벤트 체인의 정렬을 유지할 수 없는 경우, 작업 공간 소유권이 없거나 일시적인 보충 저장 +오류가 있는 경우에는 누락이 남을 수 있습니다. 기존 접근 권한 거절과 결과·binding 검증은 유지합니다. + +미측정 명령과 추가 source 분기 검토는 후속 작업입니다. 심사용·여러 프로세스 간 보충은 이번 변경 +범위 밖입니다. 일지만 고치려고 새 요청으로 다시 보내면 효과가 다시 실행될 수 있으니 저장된 결과를 +먼저 확인하세요. ## 샌드박스 연구 시험은 일부만 기록될 수 있습니다 diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 5f4681f..a2c3a9e 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -40,13 +40,26 @@ its ordinary change notification is omitted. Check the saved result before sendi A new question may use model capacity again; a newly refreshed revision guard can also permit a second change. This recovery is not a guarantee of exactly-once external effects. -## A finished request can miss some follow-up log entries - -When a request finishes, its result is saved first. The checkpoint, notification and "succeeded" -entries of the activity log are saved afterwards. If a storage error or shutdown hits between the -two, the result and its stored links stay intact and the same request returns that result, but -those follow-up entries can be missing and are not filled in later. You do not need to repeat the -request. +## Missing success-log entries can be supplemented on retry + +In LOCAL mode, retrying the same idempotency key can fill missing checkpoint, notification and +success entries for 20 measured commands and their supported saved result forms. The stored +result and its links remain unchanged. The handler, model and other effects are not run again; +once the missing entries are present, further retries add nothing. + +New entries use the actual supplementation time and carry `supplemented: true` and +`supplemented_for` with the original completion time. Existing entries are not rewritten. +An interrupted supplementation transaction rolls back its new entries and can be retried. + +This does not supplement hosted review, failed or running requests, research publication, or +restore publication. Other methods and unmeasured legacy result forms remain deferred. Unknown +or conflicting log formats, unsafe event-chain ordering (including clock rollback), missing +workspace ownership or temporary repair-storage failure can leave the gap in place. Existing +access denial and canonical result/binding checks still apply. + +Follow-up work includes the unmeasured methods and additional source-branch audits; hosted and +cross-process supplementation are outside this change. Do not send a new request just to repair +a log: it could run the effect again. Check the saved result first. ## Sandboxed research tests can leave a partial record diff --git a/src/thoth/adapters/storage/action.py b/src/thoth/adapters/storage/action.py index c576f0d..21a5f83 100644 --- a/src/thoth/adapters/storage/action.py +++ b/src/thoth/adapters/storage/action.py @@ -92,7 +92,7 @@ def list_actions(self, project_id: str) -> tuple[ActionRecord, ...]: rows = connection.execute( select(action_records) .where(action_records.c.project_id == project_id) - .order_by(action_records.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple(ActionRecord.model_validate(load(row["content_json"])) for row in rows) latest: dict[str, ActionRecord] = {} @@ -122,9 +122,7 @@ def read_action( action_records.c.action_id == action_id, ) if revision_digest is None: - statement = statement.order_by( - action_records.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(action_records.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -158,7 +156,7 @@ def list_portfolios(self, project_id: str) -> tuple[ActionPortfolioRecord, ...]: rows = connection.execute( select(action_portfolios) .where(action_portfolios.c.project_id == project_id) - .order_by(action_portfolios.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( ActionPortfolioRecord.model_validate(load(row["content_json"])) for row in rows @@ -190,9 +188,7 @@ def read_portfolio( action_portfolios.c.portfolio_id == portfolio_id, ) if revision_digest is None: - statement = statement.order_by( - action_portfolios.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(action_portfolios.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -228,7 +224,7 @@ def list_plans(self, project_id: str) -> tuple[ActionPlanRecord, ...]: rows = connection.execute( select(action_plans) .where(action_plans.c.project_id == project_id) - .order_by(action_plans.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( ActionPlanRecord.model_validate(load(row["content_json"])) for row in rows @@ -260,9 +256,7 @@ def read_plan( action_plans.c.plan_id == plan_id, ) if revision_digest is None: - statement = statement.order_by( - action_plans.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(action_plans.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: diff --git a/src/thoth/adapters/storage/control_record.py b/src/thoth/adapters/storage/control_record.py index dd0feb6..b3bf9eb 100644 --- a/src/thoth/adapters/storage/control_record.py +++ b/src/thoth/adapters/storage/control_record.py @@ -4,7 +4,7 @@ import orjson from pydantic import ValidationError -from sqlalchemy import Engine, insert, select +from sqlalchemy import Engine, Integer, insert, literal_column, select from thoth.adapters.storage.schema import control_records from thoth.adapters.storage.transaction import read_connection, write_connection @@ -55,7 +55,9 @@ def read(self, project_id: str, namespace: str, record_id: str) -> ControlRecord control_records.c.namespace == namespace, control_records.c.record_id == record_id, ) - .order_by(control_records.c.version.desc()) + .order_by( + control_records.c.version.desc(), literal_column("rowid", Integer()).desc() + ) .limit(1) ) .mappings() @@ -91,10 +93,13 @@ def list( ) if record_type is not None: statement = statement.where(control_records.c.record_type == record_type) + order = ( + (control_records.c.version, literal_column("rowid", Integer())) + if latest_only + else (control_records.c.created_at, control_records.c.version) + ) with read_connection(self._engine) as connection: - rows = connection.execute( - statement.order_by(control_records.c.created_at, control_records.c.version) - ).mappings() + rows = connection.execute(statement.order_by(*order)).mappings() parsed: list[ControlRecord] = [] for row in rows: try: diff --git a/src/thoth/adapters/storage/criterion_contract.py b/src/thoth/adapters/storage/criterion_contract.py index 5866e71..16901fe 100644 --- a/src/thoth/adapters/storage/criterion_contract.py +++ b/src/thoth/adapters/storage/criterion_contract.py @@ -140,7 +140,7 @@ def list_contracts(self, project_id: str) -> tuple[CriterionContractRecord, ...] rows = connection.execute( select(criterion_contracts) .where(criterion_contracts.c.project_id == project_id) - .order_by(criterion_contracts.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() return self._latest(tuple(self._contract(row) for row in rows)) @@ -168,9 +168,7 @@ def read_contract( if revision_digest is not None: statement = statement.where(criterion_contracts.c.revision_digest == revision_digest) else: - statement = statement.order_by( - criterion_contracts.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) with read_connection(self._engine) as connection: row = connection.execute(statement).mappings().first() return None if row is None else self._contract(row) diff --git a/src/thoth/adapters/storage/decision_object.py b/src/thoth/adapters/storage/decision_object.py index 26cde6f..e4b9b74 100644 --- a/src/thoth/adapters/storage/decision_object.py +++ b/src/thoth/adapters/storage/decision_object.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, insert, select +from sqlalchemy import Engine, Integer, insert, literal_column, select from sqlalchemy.dialects.sqlite import insert as sqlite_insert from thoth.adapters.storage.schema import ( @@ -186,7 +186,7 @@ def list_objects(self, project_id: str) -> tuple[DecisionObjectRecord, ...]: rows = connection.execute( select(decision_objects) .where(decision_objects.c.project_id == project_id) - .order_by(decision_objects.c.created_at) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( DecisionObjectRecord.model_validate(load(row["content_json"])) for row in rows @@ -209,7 +209,7 @@ def read_object( decision_objects.c.object_id == object_id, ) if revision_digest is None: - statement = statement.order_by(decision_objects.c.created_at.desc()).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(decision_objects.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -252,7 +252,7 @@ def list_relations(self, project_id: str, object_id: str) -> tuple[ObjectRelatio object_relations.c.project_id == project_id, object_relations.c.source_object_id == object_id, ) - .order_by(object_relations.c.created_at) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( ObjectRelationRecord.model_validate(load(row["content_json"])) for row in rows @@ -272,7 +272,7 @@ def read_relation(self, relation_id: str) -> ObjectRelationRecord | None: connection.execute( select(object_relations) .where(object_relations.c.relation_id == relation_id) - .order_by(object_relations.c.created_at.desc()) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() @@ -315,7 +315,9 @@ def list_attention( if object_id is not None: statement = statement.where(object_attention.c.object_id == object_id) with read_connection(self._engine) as connection: - rows = connection.execute(statement.order_by(object_attention.c.created_at)).mappings() + rows = connection.execute( + statement.order_by(literal_column("rowid", Integer())) + ).mappings() values = tuple( ObjectAttentionRecord.model_validate(load(row["content_json"])) for row in rows ) @@ -334,7 +336,7 @@ def read_attention(self, attention_id: str) -> ObjectAttentionRecord | None: connection.execute( select(object_attention) .where(object_attention.c.attention_id == attention_id) - .order_by(object_attention.c.created_at.desc()) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/events.py b/src/thoth/adapters/storage/events.py index 480a460..d1120f3 100644 --- a/src/thoth/adapters/storage/events.py +++ b/src/thoth/adapters/storage/events.py @@ -1,15 +1,20 @@ from __future__ import annotations +from collections.abc import Generator +from contextlib import contextmanager from datetime import datetime from typing import cast import orjson -from sqlalchemy import Connection, Engine, insert, select +from sqlalchemy import Connection, Engine, Integer, insert, literal_column, select +from sqlalchemy.exc import OperationalError from thoth.adapters.storage.schema import checkpoints, events from thoth.adapters.storage.transaction import read_connection, write_connection from thoth.domain.event import Checkpoint, EventRecord from thoth.ports.event_store import EventStorePort +from thoth.ports.journal import JournalCompletionUnavailable +from thoth.ports.runtime import AtomicUnitOfWorkPort def _object(value: str) -> dict[str, object]: @@ -40,6 +45,36 @@ class SqliteEventStore(EventStorePort): def __init__(self, engine: Engine) -> None: self._engine = engine + @contextmanager + def completion_transaction(self, uow: AtomicUnitOfWorkPort) -> Generator[None]: + try: + with uow.transaction(): + yield + except OperationalError as exc: + raise JournalCompletionUnavailable("JOURNAL_STORAGE_UNAVAILABLE") from exc + + def list_checkpoints(self, operation_id: str) -> tuple[Checkpoint, ...]: + with read_connection(self._engine) as connection: + rows = ( + connection.execute( + select(checkpoints) + .where(checkpoints.c.operation_id == operation_id) + .order_by(checkpoints.c.created_at, checkpoints.c.checkpoint_id) + ) + .mappings() + .all() + ) + return tuple( + Checkpoint( + checkpoint_id=str(row["checkpoint_id"]), + operation_id=str(row["operation_id"]), + payload=_object(str(row["payload_json"])), + checkpoint_digest=str(row["checkpoint_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + for row in rows + ) + def insert_event(self, event: EventRecord) -> None: with write_connection(self._engine) as connection: insert_event_on_connection(connection, event) @@ -93,7 +128,7 @@ def latest_checkpoint(self, operation_id: str) -> Checkpoint | None: connection.execute( select(checkpoints) .where(checkpoints.c.operation_id == operation_id) - .order_by(checkpoints.c.created_at.desc(), checkpoints.c.checkpoint_id.desc()) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/evidence_graph.py b/src/thoth/adapters/storage/evidence_graph.py index 7b66489..bde080f 100644 --- a/src/thoth/adapters/storage/evidence_graph.py +++ b/src/thoth/adapters/storage/evidence_graph.py @@ -93,10 +93,7 @@ def read_source_by_artifact(self, artifact_id: str) -> EvidenceSourceRecord | No connection.execute( select(evidence_sources) .where(evidence_sources.c.artifact_id == artifact_id) - .order_by( - evidence_sources.c.created_at.desc(), - literal_column("rowid", Integer()).desc(), - ) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/execution.py b/src/thoth/adapters/storage/execution.py index 3afd8d5..7f87ab3 100644 --- a/src/thoth/adapters/storage/execution.py +++ b/src/thoth/adapters/storage/execution.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, Integer, Table, insert, literal_column, select +from sqlalchemy import Engine, Integer, Table, func, insert, literal_column, select from thoth.adapters.storage.schema import ( execution_audit, @@ -74,7 +74,12 @@ def list_executions(self, project_id: str) -> tuple[PlanExecutionRecord, ...]: rows = connection.execute( select(plan_executions) .where(plan_executions.c.project_id == project_id) - .order_by(plan_executions.c.created_at, literal_column("rowid", Integer())) + .order_by( + func.coalesce( + func.json_extract(plan_executions.c.content_json, "$.revision"), 0 + ), + literal_column("rowid", Integer()), + ) ).mappings() values = tuple( PlanExecutionRecord.model_validate(load(row["content_json"])) for row in rows @@ -94,7 +99,9 @@ def read_execution(self, project_id: str, plan_execution_id: str) -> PlanExecuti plan_executions.c.plan_execution_id == plan_execution_id, ) .order_by( - plan_executions.c.created_at.desc(), + func.coalesce( + func.json_extract(plan_executions.c.content_json, "$.revision"), 0 + ).desc(), literal_column("rowid", Integer()).desc(), ) .limit(1) @@ -128,7 +135,7 @@ def list_attempts( step_execution_attempts.c.project_id == project_id, step_execution_attempts.c.plan_execution_id == plan_execution_id, ) - .order_by(step_execution_attempts.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( StepExecutionAttemptRecord.model_validate(load(row["content_json"])) for row in rows @@ -147,10 +154,7 @@ def read_attempt(self, project_id: str, attempt_id: str) -> StepExecutionAttempt step_execution_attempts.c.project_id == project_id, step_execution_attempts.c.attempt_id == attempt_id, ) - .order_by( - step_execution_attempts.c.created_at.desc(), - literal_column("rowid", Integer()).desc(), - ) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/hypothesis.py b/src/thoth/adapters/storage/hypothesis.py index 0e349a0..cd4a2dd 100644 --- a/src/thoth/adapters/storage/hypothesis.py +++ b/src/thoth/adapters/storage/hypothesis.py @@ -116,9 +116,7 @@ def read_hypothesis( hypothesis_records.c.hypothesis_id == hypothesis_id, ) if revision_digest is None: - statement = statement.order_by( - hypothesis_records.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(hypothesis_records.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -170,9 +168,7 @@ def read_portfolio( hypothesis_portfolios.c.portfolio_id == portfolio_id, ) if revision_digest is None: - statement = statement.order_by( - hypothesis_portfolios.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(hypothesis_portfolios.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -199,7 +195,7 @@ def list_relations( ) -> tuple[HypothesisRelationRecord, ...]: statement = ( select(hypothesis_relations) - .order_by(hypothesis_relations.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) .where(hypothesis_relations.c.project_id == project_id) ) if portfolio_id is not None: @@ -216,10 +212,7 @@ def read_relation(self, relation_id: str) -> HypothesisRelationRecord | None: connection.execute( select(hypothesis_relations) .where(hypothesis_relations.c.relation_id == relation_id) - .order_by( - hypothesis_relations.c.created_at.desc(), - literal_column("rowid", Integer()).desc(), - ) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() @@ -398,7 +391,7 @@ def _list[T: DomainModel]( return self._execute( select(table) .where(table.c[field] == value) - .order_by(table.c.created_at, literal_column("rowid", Integer())), + .order_by(literal_column("rowid", Integer())), model, ) diff --git a/src/thoth/adapters/storage/outcome.py b/src/thoth/adapters/storage/outcome.py index cb2dd95..290c491 100644 --- a/src/thoth/adapters/storage/outcome.py +++ b/src/thoth/adapters/storage/outcome.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, Table, insert, select +from sqlalchemy import Engine, Integer, Table, func, insert, literal_column, select from sqlalchemy.dialects.sqlite import insert as sqlite_insert from thoth.adapters.storage.schema import ( @@ -115,7 +115,12 @@ def list_series(self, project_id: str) -> tuple[OutcomeSeriesRecord, ...]: rows = connection.execute( select(outcome_series) .where(outcome_series.c.project_id == project_id) - .order_by(outcome_series.c.created_at) + .order_by( + func.coalesce( + func.json_extract(outcome_series.c.content_json, "$.revision"), 0 + ), + literal_column("rowid", Integer()), + ) ).mappings() values = tuple( OutcomeSeriesRecord.model_validate(load(row["content_json"])) for row in rows @@ -133,7 +138,12 @@ def read_series( outcome_series.c.outcome_series_id == series_id, ) if revision_digest is None: - statement = statement.order_by(outcome_series.c.created_at.desc()).limit(1) + statement = statement.order_by( + func.coalesce( + func.json_extract(outcome_series.c.content_json, "$.revision"), 0 + ).desc(), + literal_column("rowid", Integer()).desc(), + ).limit(1) else: statement = statement.where(outcome_series.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -159,7 +169,7 @@ def list_assessments(self, project_id: str) -> tuple[OutcomeAssessmentRecord, .. rows = connection.execute( select(outcome_assessments) .where(outcome_assessments.c.project_id == project_id) - .order_by(outcome_assessments.c.created_at) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( OutcomeAssessmentRecord.model_validate(load(row["content_json"])) for row in rows @@ -177,7 +187,7 @@ def read_assessment( outcome_assessments.c.outcome_assessment_id == assessment_id, ) if revision_digest is None: - statement = statement.order_by(outcome_assessments.c.created_at.desc()).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(outcome_assessments.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: diff --git a/src/thoth/adapters/storage/thread_runtime.py b/src/thoth/adapters/storage/thread_runtime.py index e41df81..d482b6f 100644 --- a/src/thoth/adapters/storage/thread_runtime.py +++ b/src/thoth/adapters/storage/thread_runtime.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, func, insert, select, update +from sqlalchemy import Engine, Integer, func, insert, literal_column, select, update from thoth.adapters.storage.schema import thread_activities, thread_checkpoints, thread_inputs from thoth.adapters.storage.transaction import read_connection, write_connection @@ -87,7 +87,7 @@ def list_checkpoints(self, project_id: str, thread_id: str) -> tuple[ThreadCheck thread_checkpoints.c.project_id == project_id, thread_checkpoints.c.thread_id == thread_id, ) - .order_by(thread_checkpoints.c.created_at, thread_checkpoints.c.checkpoint_id) + .order_by(literal_column("rowid", Integer())) ).mappings() return tuple(self._checkpoint(row) for row in rows) diff --git a/src/thoth/adapters/storage/workspace_owner.py b/src/thoth/adapters/storage/workspace_owner.py index 5a38d08..372d20e 100644 --- a/src/thoth/adapters/storage/workspace_owner.py +++ b/src/thoth/adapters/storage/workspace_owner.py @@ -190,6 +190,11 @@ def holds_workspace(workspace: Path) -> bool: return _key(workspace) in _OWNED +def workspace_owner_key(workspace: Path) -> str: + """Use the existing ownership identity for other process-local coordination.""" + return _key(workspace) + + @contextmanager def pin_workspace_owner(workspace: Path) -> Generator[bool]: """Retain only an already-owned lock; never acquire or create a lock file.""" diff --git a/src/thoth/application/commands/memory_full.py b/src/thoth/application/commands/memory_full.py index 37700b3..a0e717c 100644 --- a/src/thoth/application/commands/memory_full.py +++ b/src/thoth/application/commands/memory_full.py @@ -1,5 +1,6 @@ from __future__ import annotations +from functools import partial from typing import cast from pydantic import JsonValue @@ -42,6 +43,7 @@ projection_status_view, select_policies, ) +from thoth.application.commands.memory_records import memory_record from thoth.application.services.control_record_service import ControlRecordService from thoth.application.services.memory_router import build_recall_context from thoth.domain.control_record import ControlRecord @@ -63,6 +65,7 @@ def __init__( ) -> None: self._records = records self._controls = controls + self._memory_record = partial(memory_record, controls) self._legacy = legacy self._full = full self._ledger = ledger @@ -303,23 +306,19 @@ async def candidate_classify(self, value: dict[str, JsonValue]) -> dict[str, Jso async def validate(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: request = ValidateInput.model_validate(value) - current = self._candidate(request) - plan, reducer, verdicts, gate_payload, gate_digest = gate_draft(request, current) - gate = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="GATE", - state=reducer, - payload={**gate_payload, "gate_digest": gate_digest}, - ) - revised = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="CANDIDATE", - record_id=current.record_id, - state=reducer, - payload={**current.payload, "validated_gate_digest": gate_digest}, - ) + with self._ledger.transaction(): + current = self._candidate(request) + plan, reducer, verdicts, gate_payload, gate_digest = gate_draft(request, current) + gate = self._memory_record( + request, "GATE", reducer, {**gate_payload, "gate_digest": gate_digest} + ) + revised = self._memory_record( + request, + "CANDIDATE", + reducer, + {**current.payload, "validated_gate_digest": gate_digest}, + record_id=current.record_id, + ) return cast( dict[str, JsonValue], { @@ -334,34 +333,30 @@ async def validate(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: async def change_set_propose(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: request = ChangeSetProposeInput.model_validate(value) - candidate = self._candidate(request) - if ( - candidate.state != "VALIDATED" - or candidate.payload.get("validated_gate_digest") != request.validated_gate_digest - ): - raise RpcApplicationError( - RpcErrorCode.DOMAIN_REJECTED, "candidate gate is not validated" + with self._ledger.transaction(): + candidate = self._candidate(request) + if ( + candidate.state != "VALIDATED" + or candidate.payload.get("validated_gate_digest") != request.validated_gate_digest + ): + raise RpcApplicationError( + RpcErrorCode.DOMAIN_REJECTED, "candidate gate is not validated" + ) + entry_payload = memory_entry_payload(candidate) + memory_candidate = self._memory_record( + request, "MEMORY_ENTRY_CANDIDATE", "PROPOSED_NOT_APPLIED", entry_payload + ) + change_set = self._memory_record( + request, + "CHANGE_SET_PROPOSAL", + "PROPOSED_NOT_APPLIED", + { + "memory_candidate_digest": memory_candidate.record_digest, + "transition_intent": request.transition_intent, + "expected_memory_head_digest": request.expected_memory_head_digest, + "current_state_mutated": False, + }, ) - entry_payload = memory_entry_payload(candidate) - memory_candidate = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="MEMORY_ENTRY_CANDIDATE", - state="PROPOSED_NOT_APPLIED", - payload=entry_payload, - ) - change_set = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="CHANGE_SET_PROPOSAL", - state="PROPOSED_NOT_APPLIED", - payload={ - "memory_candidate_digest": memory_candidate.record_digest, - "transition_intent": request.transition_intent, - "expected_memory_head_digest": request.expected_memory_head_digest, - "current_state_mutated": False, - }, - ) return cast( dict[str, JsonValue], { @@ -405,28 +400,27 @@ async def revalidate(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: async def retire_propose(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: request = RetireProposeInput.model_validate(value) - current = self._memory(request) - if current.record_digest != request.expected_revision_digest: - raise RpcApplicationError(RpcErrorCode.DOMAIN_REJECTED, "memory revision changed") - candidate = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="RETIRE_CANDIDATE", - state="PROPOSED_NOT_APPLIED", - payload={ - "memory_entry_id": current.record_id, - "reason": request.reason, - "evidence_refs": request.evidence_refs, - "history_preserved": True, - }, - ) - change_set = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="CHANGE_SET_PROPOSAL", - state="PROPOSED_NOT_APPLIED", - payload={"retire_candidate_digest": candidate.record_digest}, - ) + with self._ledger.transaction(): + current = self._memory(request) + if current.record_digest != request.expected_revision_digest: + raise RpcApplicationError(RpcErrorCode.DOMAIN_REJECTED, "memory revision changed") + candidate = self._memory_record( + request, + "RETIRE_CANDIDATE", + "PROPOSED_NOT_APPLIED", + { + "memory_entry_id": current.record_id, + "reason": request.reason, + "evidence_refs": request.evidence_refs, + "history_preserved": True, + }, + ) + change_set = self._memory_record( + request, + "CHANGE_SET_PROPOSAL", + "PROPOSED_NOT_APPLIED", + {"retire_candidate_digest": candidate.record_digest}, + ) return { "retirement_candidate": candidate.model_dump(mode="json"), "change_set": change_set.model_dump(mode="json"), diff --git a/src/thoth/application/commands/memory_records.py b/src/thoth/application/commands/memory_records.py new file mode 100644 index 0000000..3f87232 --- /dev/null +++ b/src/thoth/application/commands/memory_records.py @@ -0,0 +1,24 @@ +"""Typed MEMORY record creation; callers own transaction and execution order.""" + +from thoth.application.commands.memory_inputs import ProjectInput +from thoth.application.services.control_record_service import ControlRecordService +from thoth.domain.control_record import ControlRecord + + +def memory_record( + controls: ControlRecordService, + request: ProjectInput, + kind: str, + state: str, + payload: dict[str, object], + *, + record_id: str | None = None, +) -> ControlRecord: + return controls.create( + project_id=request.project_id, + namespace="MEMORY", + record_type=kind, + state=state, + payload=payload, + record_id=record_id, + ) diff --git a/src/thoth/application/commands/reference.py b/src/thoth/application/commands/reference.py index c63da97..b27d3af 100644 --- a/src/thoth/application/commands/reference.py +++ b/src/thoth/application/commands/reference.py @@ -3,19 +3,26 @@ from pydantic import JsonValue from thoth.application.commands.thread_analysis import ThreadInputRequest +from thoth.application.services.reference_failure import reference_failure from thoth.application.services.reference_mapping import ModelReferenceMapper from thoth.application.services.reference_service import ReferenceService, ReferenceTransition from thoth.domain.evidence import InformationSufficiencyAssessment from thoth.domain.reference import ReferenceAnswerRequest, ReferenceError, ReferenceRequest +from thoth.ports.ledger import LedgerPort from thoth.ports.thread import ThreadEntryPort from thoth.protocol.jsonrpc import RpcApplicationError, RpcErrorCode class ReferenceThreadEntry: def __init__( - self, delegate: ThreadEntryPort, service: ReferenceService, mapper: ModelReferenceMapper + self, + delegate: ThreadEntryPort, + service: ReferenceService, + mapper: ModelReferenceMapper, + ledger: LedgerPort, ) -> None: self._delegate, self._service, self._mapper = delegate, service, mapper + self._ledger = ledger def authorize_before_claim(self, method: str, value: dict[str, JsonValue]) -> None: self._delegate.authorize_before_claim(method, value) @@ -45,13 +52,18 @@ async def analyze(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: selected.expected_revision_digest, ) result = await self._delegate.analyze(value) - if result.get("queued") is True or "assessment" not in result: - return result - transition = await self._reference(request, result) except ReferenceError as exc: raise RpcApplicationError( RpcErrorCode.DOMAIN_REJECTED, exc.code, data={"reason_code": exc.code} ) from exc + if result.get("queued") is True or "assessment" not in result: + return result + try: + transition = await self._reference(request, result) + except ReferenceError as exc: + raise reference_failure( + self._ledger, request.project_id, request.thread_id, result, exc.code + ) from exc if transition is None: return result return { diff --git a/src/thoth/application/commands/threads.py b/src/thoth/application/commands/threads.py index 072bb35..2a3bf20 100644 --- a/src/thoth/application/commands/threads.py +++ b/src/thoth/application/commands/threads.py @@ -247,7 +247,10 @@ async def checkpoint_list(self, value: dict[str, JsonValue]) -> dict[str, JsonVa return { "checkpoints": [ item.model_dump(mode="json") - for item in self._runtime.list_checkpoints(request.project_id, request.thread_id) + for item in sorted( + self._runtime.list_checkpoints(request.project_id, request.thread_id), + key=lambda item: (item.created_at.isoformat(), item.checkpoint_id), + ) ] } diff --git a/src/thoth/application/services/r2_closed_loop.py b/src/thoth/application/services/r2_closed_loop.py index 10f67da..28071a6 100644 --- a/src/thoth/application/services/r2_closed_loop.py +++ b/src/thoth/application/services/r2_closed_loop.py @@ -15,6 +15,10 @@ stale_head_projection, template_required_projection, ) +from thoth.application.services.r2_failure_publication import ( + AdmissionRejected, + R2FailurePublication, +) from thoth.application.services.r2_outcome_projection import ( ProcessOutcomeProjection, build_process_outcome, @@ -24,11 +28,8 @@ closed_loop_actor, failure_class_of, r2_action, - reconciliation_record, - recovery_actor, recovery_attempt, - recovery_candidate, - repair_basis_text, + recovery_draft, stage_revision, terminal_state, ) @@ -114,6 +115,7 @@ def __init__( self._clock = clock self._ids = ids self._test_lifecycle = test_lifecycle + self._failures = R2FailurePublication(ledger, sandbox) async def execute_if_required( self, @@ -152,71 +154,81 @@ async def execute_if_required( return run_hold assert bundle is not None if head_set_digest(self._ledger.read_heads(project.project_id)) != sealed_head_digest: - self._record_test_hold(prepared_test, bundle, "SANDBOX_HEAD_STALE_AFTER_IO") - return self._stale_head_execution( - project.project_id, - portfolio, - action_plan, - action.action_id, - bundle, - ) + with self._failures.transaction(): + self._record_test_hold(prepared_test, bundle, "SANDBOX_HEAD_STALE_AFTER_IO") + return self._stale_head_execution( + project.project_id, + portfolio, + action_plan, + action.action_id, + bundle, + ) failure_class = failure_class_of(bundle, recovery_policy) assessments, assessment_hold = self._measurement_assessments(prepared_test, bundle) if assessment_hold is not None: return assessment_hold - if failure_class is not None: - try: - with self._sandbox.accepting_observation(spec, bundle): - if prepared_test is not None and self._test_lifecycle is not None: - failure_completion = self._test_lifecycle.complete( - prepared_test, bundle, assessments - ) - except SandboxFailure as exc: - self._record_test_hold(prepared_test, bundle, exc.code.value) - return self._admission_hold( - project.project_id, portfolio, action_plan, action.action_id, bundle, exc - ) - attempts.append(recovery_attempt(spec, bundle, failure_class, retry_ordinal)) - if failure_class is None: - break - if ( + retry_allowed = ( failure_class == ExecutionFailureClass.TRANSIENT_INFRA and retry_ordinal < recovery_policy.max_transient_retries - ): - retry_ordinal += 1 - if prepared_test is not None and self._test_lifecycle is not None: - try: - prepared_test = self._test_lifecycle.retry(project, action, prepared_test) - except ValueError as exc: - return self._test_hold(exc, bundle=bundle) - spec, portfolio, action_plan = ( - prepared_test.spec, - prepared_test.portfolio, - prepared_test.action_plan, - ) - sealed_heads = dict(self._ledger.read_heads(project.project_id)) - sealed_head_digest = head_set_digest(sealed_heads) - research = self._research.context( - project.project_id, portfolio.object_id, sealed_heads - ) - else: - spec = spec.model_copy(update={"attempt_id": self._ids.new("r2-attempt")}) - continue - return self._failure_execution( - project=project, - action_plan=action_plan, - action_id=action.action_id, - spec=spec, - failure_class=failure_class, - attempts=tuple(attempts), - expected_heads=sealed_heads, - expected_head_set_digest=sealed_head_digest, - retry_budget_exhausted=( - failure_class == ExecutionFailureClass.TRANSIENT_INFRA - and retry_ordinal >= recovery_policy.max_transient_retries - ), - test_completion=failure_completion, ) + try: + with self._failures.final_attempt(prepared_test, failure_class, retry_allowed): + if failure_class is not None: + with self._failures.accepting_failure(spec, bundle): + if prepared_test is not None and self._test_lifecycle is not None: + failure_completion = self._test_lifecycle.complete( + prepared_test, bundle, assessments + ) + attempts.append(recovery_attempt(spec, bundle, failure_class, retry_ordinal)) + if failure_class is None: + break + if retry_allowed: + retry_ordinal += 1 + if prepared_test is not None and self._test_lifecycle is not None: + try: + prepared_test = self._test_lifecycle.retry( + project, action, prepared_test + ) + except ValueError as exc: + return self._test_hold(exc, bundle=bundle) + spec, portfolio, action_plan = ( + prepared_test.spec, + prepared_test.portfolio, + prepared_test.action_plan, + ) + sealed_heads = dict(self._ledger.read_heads(project.project_id)) + sealed_head_digest = head_set_digest(sealed_heads) + research = self._research.context( + project.project_id, portfolio.object_id, sealed_heads + ) + else: + spec = spec.model_copy( + update={"attempt_id": self._ids.new("r2-attempt")} + ) + continue + return self._failure_execution( + project=project, + action_plan=action_plan, + action_id=action.action_id, + spec=spec, + failure_class=failure_class, + attempts=tuple(attempts), + expected_heads=sealed_heads, + expected_head_set_digest=sealed_head_digest, + retry_budget_exhausted=( + failure_class == ExecutionFailureClass.TRANSIENT_INFRA + and retry_ordinal >= recovery_policy.max_transient_retries + ), + test_completion=failure_completion, + ) + except AdmissionRejected as rejected: + exc = rejected.failure + with self._failures.transaction(): + self._failures.reject_admission(bundle) + self._record_test_hold(prepared_test, bundle, exc.code.value) + return self._admission_hold( + project.project_id, portfolio, action_plan, action.action_id, bundle, exc + ) process = build_process_outcome( project=project, portfolio=portfolio, @@ -266,7 +278,9 @@ async def execute_if_required( assessments, ) except SandboxFailure as exc: - self._record_test_hold(prepared_test, bundle, exc.code.value) + with self._failures.transaction(): + self._failures.reject_admission(bundle) + self._record_test_hold(prepared_test, bundle, exc.code.value) return self._admission_hold( project.project_id, portfolio, action_plan, action.action_id, bundle, exc ) @@ -371,8 +385,9 @@ def _measurement_assessments( try: return self._test_lifecycle.prepare_assessments(prepared, bundle), None except ValueError as exc: - self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_ASSESSMENT_REJECTED") - self._record_test_hold(prepared, bundle, "RESEARCH_TEST_ASSESSMENT_REJECTED") + with self._failures.transaction(): + self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_ASSESSMENT_REJECTED") + self._record_test_hold(prepared, bundle, "RESEARCH_TEST_ASSESSMENT_REJECTED") return (), self._test_hold(exc, bundle=bundle) def _record_test_hold( @@ -424,7 +439,7 @@ def _commit_outcome( assessments: tuple[TestValidityAssessment, ...] = (), ) -> tuple[CommitResult, ResearchTestCompletion | None] | R2ClosedLoopExecution: try: - with self._sandbox.accepting_observation(spec, bundle): + with self._sandbox.accepting_observation(spec, bundle, defer_rejection=True): commit = RevisionCommitService( self._ledger, self._clock, @@ -458,8 +473,9 @@ def _commit_outcome( except Exception: if prepared_test is None: raise - self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_FINALIZATION_FAILED") - self._record_test_hold(prepared_test, bundle, "RESEARCH_TEST_FINALIZATION_FAILED") + with self._failures.transaction(): + self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_FINALIZATION_FAILED") + self._record_test_hold(prepared_test, bundle, "RESEARCH_TEST_FINALIZATION_FAILED") return self._test_hold(ValueError("RESEARCH_TEST_FINALIZATION_FAILED"), bundle=bundle) def _admission_hold( @@ -563,27 +579,12 @@ def _failure_execution( if test_completion is not None: expected_heads = dict(self._ledger.read_heads(project.project_id)) expected_head_set_digest = head_set_digest(expected_heads) - baseline_heads = dict(expected_heads) - baseline_digest = head_set_digest(baseline_heads) - repair_basis = repair_basis_text(failure_class, attempts) - candidate = recovery_candidate( - self._ids, - action_plan, - action_id, - baseline_digest, - failure_class, - repair_basis, - attempts, + draft = recovery_draft( + self._ids, action_plan, action_id, expected_heads, failure_class, attempts ) - reconciliation = None - if failure_class == ExecutionFailureClass.AMBIGUOUS_EXTERNAL: - reconciliation = reconciliation_record(self._ids, action_id, attempts) - revised_plan = action_plan.model_copy(update={"recovery_candidate": candidate}) - action_head_key = f"ACTION:{action_plan.plan_id}" - actor = recovery_actor() - research = self._research.context(project.project_id, action_plan.object_id, baseline_heads) + research = self._research.context(project.project_id, action_plan.object_id, draft.heads) batch = self._research.stage_actions( - revised_plan, actor=actor, context=research, hypotheses=research.hypotheses + draft.plan, actor=draft.actor, context=research, hypotheses=research.hypotheses ) # Recovery proposes a plan-only branch; no action/portfolio projection is activated. staged = batch.staged[0] @@ -596,27 +597,26 @@ def _failure_execution( RevisionChangeSet( changeset_id=self._ids.new("changeset"), project_id=project.project_id, - expected_heads={action_head_key: "0" * 64}, + expected_heads={draft.action_head_key: "0" * 64}, expected_head_set_digest=expected_head_set_digest, staged_revisions=(staged,), impact_plan=ImpactPropagationPlan(), - actor=actor, - reason=repair_basis, + actor=draft.actor, + reason=draft.repair_basis, ) ) baseline_after = head_set_digest(self._ledger.read_heads(project.project_id)) - terminal = terminal_state(failure_class, retry_budget_exhausted) return R2ClosedLoopExecution( projection=failure_projection( - terminal, + terminal_state(failure_class, retry_budget_exhausted), test_completion, action_id, attempts, - baseline_digest, + draft.digest, baseline_after, commit, - repair_basis, - reconciliation, + draft.repair_basis, + draft.reconciliation, retry_budget_exhausted, ) ) diff --git a/src/thoth/application/services/r2_failure_publication.py b/src/thoth/application/services/r2_failure_publication.py new file mode 100644 index 0000000..3245bb7 --- /dev/null +++ b/src/thoth/application/services/r2_failure_publication.py @@ -0,0 +1,56 @@ +"""Short failure publication boundaries, entered only after external sandbox work.""" + +from collections.abc import Generator +from contextlib import contextmanager + +from thoth.application.services.r2_test_lifecycle import PreparedResearchExecution +from thoth.application.services.sandbox_service import SandboxExecutionBundle, SandboxService +from thoth.domain.recovery import ExecutionFailureClass +from thoth.domain.sandbox import SandboxFailure, SandboxRunSpec +from thoth.ports.ledger import LedgerPort + + +class AdmissionRejected(Exception): + """Only an acceptance failure; a later recovery-candidate error is never wrapped.""" + + def __init__(self, failure: SandboxFailure) -> None: + super().__init__(str(failure)) + self.failure = failure + + +class R2FailurePublication: + def __init__(self, ledger: LedgerPort, sandbox: SandboxService) -> None: + self._ledger = ledger + self._sandbox = sandbox + + @contextmanager + def transaction(self) -> Generator[None]: + with self._ledger.transaction(): + yield + + @contextmanager + def final_attempt( + self, + prepared: PreparedResearchExecution | None, + failure_class: ExecutionFailureClass | None, + retry_allowed: bool, + ) -> Generator[None]: + if prepared is not None and failure_class is not None and not retry_allowed: + with self._ledger.transaction(): + yield + else: + # Plain execution, success and retry paths keep their existing boundaries. + yield + + @contextmanager + def accepting_failure( + self, spec: SandboxRunSpec, bundle: SandboxExecutionBundle + ) -> Generator[None]: + try: + with self._sandbox.accepting_observation(spec, bundle, defer_rejection=True): + yield + except SandboxFailure as failure: + raise AdmissionRejected(failure) from failure + + def reject_admission(self, bundle: SandboxExecutionBundle) -> None: + self._sandbox.reject_observation(bundle, reason="SANDBOX_ADMISSION_BASIS_CHANGED") diff --git a/src/thoth/application/services/r2_recovery_records.py b/src/thoth/application/services/r2_recovery_records.py index 23a8d20..5e753c4 100644 --- a/src/thoth/application/services/r2_recovery_records.py +++ b/src/thoth/application/services/r2_recovery_records.py @@ -3,19 +3,20 @@ These functions build the values the coordinator stages, links or reports: the R2 action pick, a recovery attempt, the linked outcome, the two coordinator actors, failure classification, the recovery candidate and reconciliation records, and the staged revision of one entity. They read -nothing and write nothing. The coordinator keeps every await, every ledger, sandbox, research and -lifecycle call and every transaction, and passes the id generator and the clock through as the -original objects so ids and times are drawn in the same order as before. +nothing and write nothing to storage. The coordinator and failure publisher retain every await, +ledger, sandbox, research, lifecycle call and transaction. The original id generator and clock +are passed through, so ids and times are drawn in the same order as before. """ from __future__ import annotations +from dataclasses import dataclass from typing import cast from thoth.application.services.sandbox_service import SandboxExecutionBundle from thoth.domain.action import ActionCandidate, ActionPlan from thoth.domain.actor import ActorRef -from thoth.domain.canonical import canonical_payload, domain_digest +from thoth.domain.canonical import canonical_payload, domain_digest, head_set_digest from thoth.domain.enums import ActorKind, EntityType, RiskTier from thoth.domain.outcome import OutcomeRecord from thoth.domain.policy import R2RecoveryPolicy @@ -197,6 +198,48 @@ def reconciliation_record( return reconciliation +@dataclass(frozen=True) +class RecoveryDraft: + heads: dict[str, str] + digest: str + repair_basis: str + plan: ActionPlan + reconciliation: ReconciliationRecord | None + action_head_key: str + actor: ActorRef + + +def recovery_draft( + ids: IdGeneratorPort, + action_plan: ActionPlan, + action_id: str, + expected_heads: dict[str, str], + failure_class: ExecutionFailureClass, + attempts: tuple[RecoveryAttempt, ...], +) -> RecoveryDraft: + baseline_heads = dict(expected_heads) + baseline_digest = head_set_digest(baseline_heads) + repair_basis = repair_basis_text(failure_class, attempts) + candidate = recovery_candidate( + ids, action_plan, action_id, baseline_digest, failure_class, repair_basis, attempts + ) + reconciliation = None + if failure_class == ExecutionFailureClass.AMBIGUOUS_EXTERNAL: + reconciliation = reconciliation_record(ids, action_id, attempts) + revised_plan = action_plan.model_copy(update={"recovery_candidate": candidate}) + action_head_key = f"ACTION:{action_plan.plan_id}" + actor = recovery_actor() + return RecoveryDraft( + baseline_heads, + baseline_digest, + repair_basis, + revised_plan, + reconciliation, + action_head_key, + actor, + ) + + def terminal_state(failure_class: ExecutionFailureClass, retry_budget_exhausted: bool) -> str: terminal = ( "RECONCILIATION_REQUIRED" diff --git a/src/thoth/application/services/reference_failure.py b/src/thoth/application/services/reference_failure.py new file mode 100644 index 0000000..4d02915 --- /dev/null +++ b/src/thoth/application/services/reference_failure.py @@ -0,0 +1,112 @@ +"""Trusted reference-only notices, after verifying the returned research commit is readable.""" + +import re + +from pydantic import JsonValue, ValidationError + +from thoth.application.services.revision_service import CommitResult +from thoth.domain.research_execution import research_work +from thoth.domain.research_failure import FailureOrigin, ResearchFailureCause, failure_cause +from thoth.ports.ledger import LedgerPort +from thoth.protocol.deferred import current_operation +from thoth.protocol.jsonrpc import RpcApplicationError, RpcErrorCode + + +class SavedReferenceFailure(RpcApplicationError): + def __init__(self, reason_code: str, *, intermediate: bool) -> None: + if re.fullmatch(r"REFERENCE_[A-Z0-9_]{1,149}", reason_code) is None: + raise ValueError("REFERENCE_NOTICE_REASON_INVALID") + self.reason_code = reason_code + ko = "연구의 중간 기록은 저장됐습니다." if intermediate else "연구 결과는 저장됐습니다." + en = ( + "Intermediate research records were saved." + if intermediate + else "The research result was saved." + ) + message = ( + f"{ko} 기준(reference) 단계만 실패했습니다 (사유: {reason_code}). " + "결과는 기록에서 볼 수 있습니다. 같은 질문을 새로 보내면 연구가 다시 실행될 수 있으니 " + "먼저 저장된 결과를 확인하세요. / " + f"{en} Only the reference step failed (reason: {reason_code}). " + "You can read the result in the history. Sending the same question again as a new " + "request may run the research again, so check the saved result first." + ) + super().__init__( + RpcErrorCode.DOMAIN_REJECTED, + message, + data={"reason_code": reason_code, "research_saved": True}, + ) + + +def reference_failure_cause( + error: Exception, origin: FailureOrigin +) -> tuple[ResearchFailureCause, SavedReferenceFailure | None]: + primary = failure_cause(error, origin) + notice = error if isinstance(error, SavedReferenceFailure) else None + if notice is not None: + primary = primary.model_copy( + update={ + "reason_code": notice.reason_code, + "exception_type": RpcApplicationError.__name__, + } + ) + return primary, notice + + +def research_commit_is_readable( + ledger: LedgerPort, project_id: str, thread_id: str, result: dict[str, JsonValue] +) -> bool: + operation = current_operation.get() + if operation is None or operation.project_id != project_id: + return False + try: + commit = CommitResult.model_validate(result.get("commit")) + except ValidationError: + return False + if ( + commit.receipt.project_id != project_id + or commit.receipt.recorded_at < operation.created_at + or not commit.committed_revision_ids + or not set(commit.committed_revision_ids).issubset(commit.receipt.subject_refs) + ): + return False + work = research_work.get() + if work is not None and ( + work.request_ref.project_id != project_id + or work.request_ref.entity_id != f"request:{thread_id}" + or commit.receipt.receipt_id not in work.produced_receipt_refs + or not set(commit.committed_revision_ids).issubset( + ref.revision_id for ref in work.produced_refs + ) + ): + return False + if commit.receipt not in ledger.read_receipts(project_id): + return False + for identifier in commit.committed_revision_ids: + revision = ledger.read_revision_by_id(project_id, identifier) + if revision is None or revision.project_id != project_id: + return False + snapshot = ledger.read_snapshot(revision.snapshot_id) + if snapshot is None or ( + snapshot.project_id, + snapshot.entity_type, + snapshot.entity_id, + ) != (project_id, revision.entity_type, revision.entity_id): + return False + return True + + +def reference_failure( + ledger: LedgerPort, + project_id: str, + thread_id: str, + result: dict[str, JsonValue], + reason_code: str, +) -> RpcApplicationError: + if re.fullmatch( + r"REFERENCE_[A-Z0-9_]{1,149}", reason_code + ) is not None and research_commit_is_readable(ledger, project_id, thread_id, result): + return SavedReferenceFailure(reason_code, intermediate=research_work.get() is not None) + return RpcApplicationError( + RpcErrorCode.DOMAIN_REJECTED, reason_code, data={"reason_code": reason_code} + ) diff --git a/src/thoth/application/services/research_attempt_runner.py b/src/thoth/application/services/research_attempt_runner.py index bfafd46..1b9ee43 100644 --- a/src/thoth/application/services/research_attempt_runner.py +++ b/src/thoth/application/services/research_attempt_runner.py @@ -10,6 +10,7 @@ from thoth.application.services.model_outcomes import record_model_outcome from thoth.application.services.post_execution_resume import resume_post_execution_learning +from thoth.application.services.reference_failure import reference_failure_cause from thoth.application.services.request_language import apply_request_language from thoth.application.services.research_boundary import RequestBoundary from thoth.application.services.research_termination import ResearchTerminationService @@ -181,7 +182,7 @@ def terminate( ) -> PendingExecution: from thoth.domain.research_failure import FailureOrigin - primary = failure_cause(primary_error, cast(FailureOrigin, origin)) + primary, notice = reference_failure_cause(primary_error, cast(FailureOrigin, origin)) secondary = ( None if secondary_error is None else failure_cause(secondary_error, "HOLD_PUBLICATION") ) @@ -194,7 +195,7 @@ def terminate( ) try: ResearchTerminationService(host.records, host.operations, host.threads).finish_failure( - attempt, request, primary, secondary + attempt, request, primary, secondary, notice=notice ) except ResearchLeaseLost: return PendingExecution({"status": "ATTEMPT_FENCED"}) diff --git a/src/thoth/application/services/research_termination.py b/src/thoth/application/services/research_termination.py index 6cf2369..48c0447 100644 --- a/src/thoth/application/services/research_termination.py +++ b/src/thoth/application/services/research_termination.py @@ -4,6 +4,7 @@ from pydantic import JsonValue +from thoth.application.services.reference_failure import SavedReferenceFailure from thoth.application.services.request_records import RequestRecords from thoth.domain.enums import EntityType, OperationState, ThreadExecutionState from thoth.domain.research_failure import ResearchFailureCause, ResearchFailureRecord @@ -25,6 +26,8 @@ def finish_failure( request: ThreadRequestRevision, primary: ResearchFailureCause, secondary: ResearchFailureCause | None = None, + *, + notice: SavedReferenceFailure | None = None, ) -> ResearchFailureRecord | None: project, op_id = request.project_id, attempt.operation_id key = f"failure:{op_id}:{attempt.attempt_epoch}" @@ -82,10 +85,13 @@ def finish_failure( dict[str, JsonValue], { "code": -32603, - "message": "research execution failed", + "message": "research execution failed" + if notice is None + else notice.message, "data": { "reason_code": primary.reason_code, "failure": failure.model_dump(mode="json"), + **({"research_saved": True} if notice is not None else {}), }, }, ), diff --git a/src/thoth/application/services/sandbox_service.py b/src/thoth/application/services/sandbox_service.py index df61697..b315624 100644 --- a/src/thoth/application/services/sandbox_service.py +++ b/src/thoth/application/services/sandbox_service.py @@ -327,6 +327,8 @@ def accepting_observation( self, spec: SandboxRunSpec, bundle: SandboxExecutionBundle, + *, + defer_rejection: bool = False, ) -> Generator[ArtifactEnvelope, None, None]: try: with self._ledger.transaction(): @@ -385,7 +387,8 @@ def accepting_observation( }, ) except SandboxFailure: - self.reject_observation(bundle, reason="SANDBOX_ADMISSION_BASIS_CHANGED") + if not defer_rejection: + self.reject_observation(bundle, reason="SANDBOX_ADMISSION_BASIS_CHANGED") raise def reject_observation(self, bundle: SandboxExecutionBundle, *, reason: str) -> None: diff --git a/src/thoth/apps/reference_composition.py b/src/thoth/apps/reference_composition.py index 1ff9c54..937309b 100644 --- a/src/thoth/apps/reference_composition.py +++ b/src/thoth/apps/reference_composition.py @@ -40,4 +40,5 @@ def reference_thread_entry( ids=ids, ), ModelReferenceMapper(models, projects, ledger), + ledger, ) diff --git a/src/thoth/apps/resource_scope_composition.py b/src/thoth/apps/resource_scope_composition.py index 025b6b6..5a834a5 100644 --- a/src/thoth/apps/resource_scope_composition.py +++ b/src/thoth/apps/resource_scope_composition.py @@ -31,6 +31,7 @@ from thoth.ports.sandbox import SandboxPort from thoth.ports.store_bundle import StoreBundlePort from thoth.protocol.bus import CommandBus +from thoth.protocol.journal_completion import JournalCompletion from thoth.protocol.operation_owner import OperationOwner from thoth.protocol.registry import MethodRegistry @@ -117,6 +118,7 @@ def scope_bus( queued_admission: Callable[[OperationRecord], dict[str, JsonValue] | None] | None = None, owner: OperationOwner | None = None, journal: JournalPort | None = None, + journal_completion: JournalCompletion | None = None, ) -> CommandBus: handlers = OperationCommandHandlers( operations, @@ -143,6 +145,7 @@ def scope_bus( before_continue=before_continue, queued_admission=queued_admission, owner=owner, + journal_completion=journal_completion, ) diff --git a/src/thoth/apps/runtime_entry.py b/src/thoth/apps/runtime_entry.py index 30c7601..c31cfc9 100644 --- a/src/thoth/apps/runtime_entry.py +++ b/src/thoth/apps/runtime_entry.py @@ -6,7 +6,11 @@ from pathlib import Path from thoth.adapters.operation_host_identity import operation_host_token -from thoth.adapters.storage.workspace_owner import holds_workspace, pin_workspace_owner +from thoth.adapters.storage.workspace_owner import ( + holds_workspace, + pin_workspace_owner, + workspace_owner_key, +) from thoth.application.commands import ThreadCommandHandlers from thoth.application.commands.research_operations import register_research_operations from thoth.application.services import FieldMeasurementService @@ -22,6 +26,7 @@ from thoth.domain.deployment_mode import DeploymentMode from thoth.ports.criterion_contract import CriterionContractStorePort from thoth.ports.evidence_graph import EvidenceGraphStorePort +from thoth.ports.journal import JournalCompletionStorePort from thoth.ports.ledger import ManagedLedgerPort from thoth.ports.model import ModelResolverPort from thoth.ports.model_catalog import ModelCatalogPort @@ -30,6 +35,7 @@ from thoth.ports.store_bundle import StoreBundlePort from thoth.ports.task_profile import TaskProfileCatalogPort from thoth.protocol.bus import CommandBus +from thoth.protocol.journal_completion import JournalCompletion from thoth.protocol.operation_owner import OperationOwner @@ -84,6 +90,7 @@ def assemble_runtime_entry( before_continue = wait_for_hosted_dispatch journal = OperationJournal(stores.events, clock, ids) owner = None + journal_completion = None if mode is DeploymentMode.LOCAL: owner = OperationOwner( worker=research_component.worker, @@ -96,6 +103,16 @@ def assemble_runtime_entry( clock=clock, resource_access=artifact_ledger.scopes, ) + if isinstance(stores.events, JournalCompletionStorePort): + journal_completion = JournalCompletion( + workspace_key=workspace_owner_key(workspace), + pin_workspace=partial(pin_workspace_owner, workspace), + operations=operations, + journal=journal, + store=stores.events, + uow=stores.immediate_uow, + resource_access=artifact_ledger.scopes, + ) bus = scope_bus( registration.registry, ledger, @@ -109,6 +126,7 @@ def assemble_runtime_entry( before_continue=before_continue, queued_admission=research_component.entry.queued_admission, owner=owner, + journal_completion=journal_completion, journal=journal, ) registration.judgment_review.connect(bus, research_component.entry) diff --git a/src/thoth/ports/journal.py b/src/thoth/ports/journal.py index 6c6f05b..f49c358 100644 --- a/src/thoth/ports/journal.py +++ b/src/thoth/ports/journal.py @@ -1,11 +1,13 @@ from __future__ import annotations -from typing import Protocol +from contextlib import AbstractContextManager +from typing import Protocol, runtime_checkable from pydantic import JsonValue from thoth.domain.event import Checkpoint, EventRecord from thoth.domain.operation import OperationRecord +from thoth.ports.runtime import AtomicUnitOfWorkPort class JournalPort(Protocol): @@ -23,3 +25,16 @@ def append( ) -> EventRecord: ... def checkpoint(self, *, operation_id: str, payload: dict[str, JsonValue]) -> Checkpoint: ... + + +class JournalCompletionUnavailable(RuntimeError): + """A repair storage failure; the existing successful result remains authoritative.""" + + +@runtime_checkable +class JournalCompletionStorePort(Protocol): + """Optional complete journal reads and repair-only infrastructure error boundary.""" + + def list_events(self, operation_id: str) -> tuple[EventRecord, ...]: ... + def list_checkpoints(self, operation_id: str) -> tuple[Checkpoint, ...]: ... + def completion_transaction(self, uow: AtomicUnitOfWorkPort) -> AbstractContextManager[None]: ... diff --git a/src/thoth/protocol/bus.py b/src/thoth/protocol/bus.py index 2ebf574..92196c0 100644 --- a/src/thoth/protocol/bus.py +++ b/src/thoth/protocol/bus.py @@ -43,6 +43,11 @@ PendingExecution, current_operation, ) +from thoth.protocol.journal_completion import ( + JournalCompletion, + journal_tail_guard, + operation_result_digest, +) from thoth.protocol.jsonrpc import ( JsonRpcError, JsonRpcRequest, @@ -53,6 +58,7 @@ from thoth.protocol.notifications import notifications_for from thoth.protocol.operation_claim import claim_busy_response, claim_operation from thoth.protocol.operation_owner import OperationOwner, stale_running_error +from thoth.protocol.operation_scope import SYSTEM_PROJECT_METHODS from thoth.protocol.read_queries import READ_QUERY_METHODS from thoth.protocol.registry import CommandHandler, MethodRegistry @@ -84,6 +90,7 @@ def __init__( before_continue: Callable[[str], Awaitable[None]] | None = None, queued_admission: Callable[[OperationRecord], dict[str, JsonValue] | None] | None = None, owner: OperationOwner | None = None, + journal_completion: JournalCompletion | None = None, ) -> None: self._registry = registry self._operations = operations @@ -96,6 +103,7 @@ def __init__( self._before_continue = before_continue self._queued_admission = queued_admission self._owner = owner + self._journal_completion = journal_completion self._operation_task_canceller: Callable[[str], None] | None = None self._failure_diagnostics = ( operations if isinstance(operations, InternalFailureDiagnosticPort) else None @@ -454,30 +462,31 @@ async def _execute(self, ticket: DispatchTicket) -> JsonRpcResponse: ) self._measure(request, operation.project_id, "FAILED") return JsonRpcResponse(id=request.id, error=error) - completed = self._operations.complete( - operation.operation_id, - durable_value, - completed_at=self._clock.now(), - ) - if completed.state is not OperationState.SUCCEEDED: - return self._replay(request.id, completed) - result_digest = domain_digest("OPERATION_RESULT", "1.0.0", canonical_payload(durable_value)) - if self._journal is not None: - self._journal.checkpoint( - operation_id=operation.operation_id, - payload={"state": completed.state.value, "result_digest": result_digest}, - ) - for notification in notifications_for(request.method, durable_value): + with journal_tail_guard(self._journal_completion, operation): + completed = self._operations.complete( + operation.operation_id, + durable_value, + completed_at=self._clock.now(), + ) + if completed.state is not OperationState.SUCCEEDED: + return self._replay(request.id, completed) + result_digest = operation_result_digest(durable_value) + if self._journal is not None: + self._journal.checkpoint( + operation_id=operation.operation_id, + payload={"state": completed.state.value, "result_digest": result_digest}, + ) + for notification in notifications_for(request.method, durable_value): + self._append_event( + operation, + notification, + {"method": request.method, "result_digest": result_digest}, + ) self._append_event( operation, - notification, - {"method": request.method, "result_digest": result_digest}, + "operation.succeeded", + {"result_digest": result_digest}, ) - self._append_event( - operation, - "operation.succeeded", - {"result_digest": result_digest}, - ) self._measure(request, operation.project_id, "SUCCEEDED") response = self._result_payload(completed) if isinstance(value, EphemeralCommandResult): @@ -499,18 +508,7 @@ def _measure(self, request: JsonRpcRequest, project_id: str, outcome: str) -> No return def _project_scope(self, request: JsonRpcRequest) -> str: - if request.method in { - "workspace/setup/read", - "workspace/setup/update", - "workspace/ready", - "model/credential/list", - "model/credential/register", - "model/credential/login/status", - "model/credential/login/cancel", - "model/credential/login/complete", - "model/catalog/refresh", - "model/tooling/install", - }: + if request.method in SYSTEM_PROJECT_METHODS: return "system:workspace" value = request.params.input.get("project_id") if not isinstance(value, str) or not value: @@ -607,6 +605,8 @@ def _fail_abandoned_running( def _replay(self, request_id: str | int, operation: OperationRecord) -> JsonRpcResponse: if self._resource_access is not None: self._resource_access.require_operation(operation) + if self._journal_completion is not None: + operation = self._journal_completion.repair(operation) if operation.state == OperationState.SUCCEEDED and operation.result is not None: return JsonRpcResponse(id=request_id, result=self._result_payload(operation)) if operation.state == OperationState.FAILED and operation.error is not None: diff --git a/src/thoth/protocol/journal_completion.py b/src/thoth/protocol/journal_completion.py new file mode 100644 index 0000000..0c99261 --- /dev/null +++ b/src/thoth/protocol/journal_completion.py @@ -0,0 +1,242 @@ +"""Supplement a measured success tail without repeating effects or changing its result.""" + +from collections import Counter +from collections.abc import Callable, Generator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from typing import Literal + +from pydantic import JsonValue + +from thoth.domain.canonical import canonical_payload, domain_digest +from thoth.domain.enums import OperationState +from thoth.domain.event import EventRecord +from thoth.domain.operation import OperationRecord +from thoth.domain.resource_scope import ResourceScopeError, resource_use_verification +from thoth.ports.journal import ( + JournalCompletionStorePort, + JournalCompletionUnavailable, + JournalPort, +) +from thoth.ports.operation import OperationStorePort +from thoth.ports.resource_scope import ResourceAccessPort +from thoth.ports.runtime import AtomicUnitOfWorkPort +from thoth.protocol.journal_completion_gate import journal_gate +from thoth.protocol.journal_completion_methods import SUPPLEMENTABLE_METHODS, supported_result +from thoth.protocol.notifications import notifications_for + + +def operation_result_digest(value: dict[str, JsonValue]) -> str: + return domain_digest("OPERATION_RESULT", "1.0.0", canonical_payload(value)) + + +@dataclass(frozen=True) +class CompletionOutcome: + operation: OperationRecord + status: Literal["NOT_ELIGIBLE", "COMPLETE", "SUPPLEMENTED", "DEFERRED"] + added: int = 0 + reason: str | None = None + + +@dataclass(frozen=True) +class _Missing: + checkpoint: bool + notifications: tuple[str, ...] + succeeded: bool + + @property + def count(self) -> int: + return int(self.checkpoint) + len(self.notifications) + int(self.succeeded) + + +class _Deferred(Exception): + pass + + +def _payload_matches( + actual: dict[str, object], base: dict[str, JsonValue], completed_at: str +) -> bool: + if actual == base: + return True + return ( + actual == {**base, "supplemented": True, "supplemented_for": completed_at} + and actual.get("supplemented") is True + ) + + +def _chain_valid(events: tuple[EventRecord, ...], operation: OperationRecord) -> bool: + if len({event.event_id for event in events}) != len(events): + return False + previous = None + for event in events: + if ( + event.operation_id != operation.operation_id + or event.project_id != operation.project_id + or event.schema_version != "1.0.0" + or event.previous_event_digest != previous + or event.event_digest + != domain_digest( + "OPERATION_EVENT", + "1.0.0", + canonical_payload(event.model_dump(mode="python", exclude={"event_digest"})), + ) + ): + return False + previous = event.event_digest + return True + + +class JournalCompletion: + def __init__( + self, + *, + workspace_key: str, + pin_workspace: Callable[[], AbstractContextManager[bool]], + operations: OperationStorePort, + journal: JournalPort, + store: JournalCompletionStorePort, + uow: AtomicUnitOfWorkPort, + resource_access: ResourceAccessPort, + ) -> None: + self.workspace_key = workspace_key + self._pin = pin_workspace + self._operations = operations + self._journal = journal + self._store = store + self._uow = uow + self._access = resource_access + + def repair(self, operation: OperationRecord) -> OperationRecord: + return self.attempt(operation).operation + + def attempt(self, operation: OperationRecord) -> CompletionOutcome: + if ( + operation.state is not OperationState.SUCCEEDED + or operation.method not in SUPPLEMENTABLE_METHODS + ): + return CompletionOutcome(operation, "NOT_ELIGIBLE") + if ( + operation.result is None + or operation.completed_at is None + or operation.error is not None + or operation.schema_version != "1.0.0" + or not supported_result(operation.method, operation.result) + ): + return CompletionOutcome(operation, "DEFERRED", reason="UNSUPPORTED_SAVED_FORM") + with journal_gate(self.workspace_key, operation.operation_id), self._pin() as held: + if not held: + return CompletionOutcome(operation, "DEFERRED", reason="WORKSPACE_NOT_OWNED") + try: + with ( + resource_use_verification(operation.project_id), + self._store.completion_transaction(self._uow), + ): + current = self._operations.read(operation.operation_id) + if current is None: + raise ResourceScopeError("OPERATION_RESOURCE_BINDING_INVALID") + self._access.require_operation(current) + if current.model_dump(mode="json") != operation.model_dump(mode="json"): + raise ResourceScopeError("OPERATION_RESOURCE_BINDING_INVALID") + missing = self._inspect(current) + if not missing.count: + return CompletionOutcome(operation, "COMPLETE") + self._write(current, missing) + if self._inspect(current).count: + raise _Deferred("INCOMPLETE_AFTER_APPEND") + return CompletionOutcome(operation, "SUPPLEMENTED", added=missing.count) + except _Deferred as exc: + return CompletionOutcome(operation, "DEFERRED", reason=str(exc)) + except JournalCompletionUnavailable: + return CompletionOutcome(operation, "DEFERRED", reason="STORAGE_UNAVAILABLE") + + def _inspect(self, operation: OperationRecord) -> _Missing: + assert operation.result is not None and operation.completed_at is not None + digest = operation_result_digest(operation.result) + completed_at = operation.completed_at.isoformat() + expected = notifications_for(operation.method, operation.result) + events = self._store.list_events(operation.operation_id) + points = self._store.list_checkpoints(operation.operation_id) + if not events: + raise _Deferred("MISSING_STARTED") + if not _chain_valid(events, operation): + raise _Deferred("UNSAFE_EVENT_CHAIN_OR_ORDER") + if events[0].event_type != "operation.started": + raise _Deferred("MISSING_STARTED") + counts = Counter(event.event_type for event in events) + if any(count != 1 for count in counts.values()): + raise _Deferred("DUPLICATE_EVENT") + for event in events: + if event.event_type == "operation.started": + if event.payload != {"method": operation.method}: + raise _Deferred("UNSUPPORTED_STARTED_PAYLOAD") + continue + if event.event_type == "operation.succeeded": + base: dict[str, JsonValue] = {"result_digest": digest} + elif event.event_type in expected: + base = {"method": operation.method, "result_digest": digest} + else: + raise _Deferred("OTHER_PUBLISHER_EVENT") + if not _payload_matches(event.payload, base, completed_at): + raise _Deferred("EVENT_PAYLOAD_MISMATCH") + if len(points) > 1: + raise _Deferred("MULTIPLE_CHECKPOINTS") + if points: + point = points[0] + if ( + point.operation_id != operation.operation_id + or not _payload_matches( + point.payload, {"state": "SUCCEEDED", "result_digest": digest}, completed_at + ) + or point.checkpoint_digest + != domain_digest( + "OPERATION_CHECKPOINT", + "1.0.0", + canonical_payload( + point.model_dump(mode="python", exclude={"checkpoint_digest"}) + ), + ) + ): + raise _Deferred("CHECKPOINT_PAYLOAD_MISMATCH") + return _Missing( + not points, + tuple(name for name in expected if name not in counts), + "operation.succeeded" not in counts, + ) + + def _write(self, operation: OperationRecord, missing: _Missing) -> None: + assert operation.result is not None and operation.completed_at is not None + digest = operation_result_digest(operation.result) + mark: dict[str, JsonValue] = { + "supplemented": True, + "supplemented_for": operation.completed_at.isoformat(), + } + if missing.checkpoint: + self._journal.checkpoint( + operation_id=operation.operation_id, + payload={"state": "SUCCEEDED", "result_digest": digest, **mark}, + ) + for notification in missing.notifications: + self._journal.append( + project_id=operation.project_id, + operation_id=operation.operation_id, + event_type=notification, + payload={"method": operation.method, "result_digest": digest, **mark}, + ) + if missing.succeeded: + self._journal.append( + project_id=operation.project_id, + operation_id=operation.operation_id, + event_type="operation.succeeded", + payload={"result_digest": digest, **mark}, + ) + + +@contextmanager +def journal_tail_guard( + completion: JournalCompletion | None, operation: OperationRecord +) -> Generator[None]: + if completion is None or operation.method not in SUPPLEMENTABLE_METHODS: + yield + else: + with journal_gate(completion.workspace_key, operation.operation_id): + yield diff --git a/src/thoth/protocol/journal_completion_gate.py b/src/thoth/protocol/journal_completion_gate.py new file mode 100644 index 0000000..fcc89df --- /dev/null +++ b/src/thoth/protocol/journal_completion_gate.py @@ -0,0 +1,34 @@ +"""Reentrant success-tail gates shared by LOCAL runtimes in one process.""" + +from _thread import RLock +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from threading import Lock + + +@dataclass +class _Entry: + lock: RLock = field(default_factory=RLock) + users: int = 0 + + +_entries: dict[tuple[str, str], _Entry] = {} +_registry_lock = Lock() + + +@contextmanager +def journal_gate(workspace_key: str, operation_id: str) -> Generator[None]: + """Count holders and waiters, then discard the entry after the last exit.""" + key = (workspace_key, operation_id) + with _registry_lock: + entry = _entries.setdefault(key, _Entry()) + entry.users += 1 + try: + with entry.lock: + yield + finally: + with _registry_lock: + entry.users -= 1 + if entry.users == 0: + del _entries[key] diff --git a/src/thoth/protocol/journal_completion_methods.py b/src/thoth/protocol/journal_completion_methods.py new file mode 100644 index 0000000..5e7a59f --- /dev/null +++ b/src/thoth/protocol/journal_completion_methods.py @@ -0,0 +1,213 @@ +"""V-J measured durable result forms; unknown methods and forms are not repaired.""" + +from pydantic import JsonValue + +_FORM_1 = frozenset( + { + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_2 = frozenset( + { + "cutoff_at", + "description", + "impact", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_3 = frozenset( + { + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "reference", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_4 = frozenset( + { + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "role", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_5 = frozenset( + { + "baseline_candidate", + "impact", + "protected_authority_package", + } +) + +_FORM_6 = frozenset( + { + "branch", + "content_mutated", + } +) + +_FORM_7 = frozenset( + { + "change_set", + "conflict_state", + "missing_dependencies", + } +) + +_FORM_8 = frozenset( + { + "change_set", + "checks", + "impact_propagation_plan", + "state", + "validation_bundle_digest", + } +) + +_FORM_9 = frozenset( + { + "change_set", + "impact_propagation_plan", + "state", + "validation_bundle_digest", + } +) + +_FORM_10 = frozenset( + { + "conflict", + "head_mutated", + "merge_proposal", + "receipt", + "semantic_merge", + } +) + +_FORM_11 = frozenset( + { + "candidate_revision", + "change_set", + "head_mutated", + "merge_proposal", + } +) + +_FORM_12 = frozenset( + { + "impact_preview", + "proposal", + "staged_snapshot", + "validation_state", + } +) + +_FORM_13 = frozenset( + { + "recompute_task", + } +) + +_FORM_14 = frozenset( + { + "change_set", + "external_effect_rollback_claimed", + "impact_plan", + "restore_proposal", + } +) + +SUPPORTED_RESULT_FORMS: dict[str, tuple[frozenset[str], ...]] = { + "project/activate": (_FORM_1,), + "project/archive": (_FORM_1,), + "project/create": (_FORM_1,), + "project/cutoff/update": (_FORM_2,), + "project/metadata/update": (_FORM_1,), + "project/overlay/update": (_FORM_1,), + "project/policy/update": (_FORM_1,), + "project/reference/import": (_FORM_3,), + "project/restore": (_FORM_1,), + "project/role/assign": (_FORM_4,), + "project/role/revoke": (_FORM_4,), + "revision/baseline/prepare": (_FORM_5,), + "revision/branch/create": (_FORM_6,), + "revision/changeSet/create": (_FORM_7,), + "revision/changeSet/validate": ( + _FORM_8, + _FORM_9, + ), + "revision/merge/propose": (_FORM_10,), + "revision/merge/resolve": (_FORM_11,), + "revision/propose": (_FORM_12,), + "revision/recompute/request": (_FORM_13,), + "revision/restore/propose": (_FORM_14,), +} +SUPPLEMENTABLE_METHODS = frozenset(SUPPORTED_RESULT_FORMS) + + +def supported_result(method: str, result: dict[str, JsonValue]) -> bool: + return ( + frozenset(result) in SUPPORTED_RESULT_FORMS.get(method, ()) + and result.get("schema_version", "1.0.0") == "1.0.0" + ) diff --git a/src/thoth/protocol/operation_scope.py b/src/thoth/protocol/operation_scope.py new file mode 100644 index 0000000..3e3105d --- /dev/null +++ b/src/thoth/protocol/operation_scope.py @@ -0,0 +1,16 @@ +"""Existing system RPCs that do not require input.project_id.""" + +SYSTEM_PROJECT_METHODS = frozenset( + { + "workspace/setup/read", + "workspace/setup/update", + "workspace/ready", + "model/credential/list", + "model/credential/register", + "model/credential/login/status", + "model/credential/login/cancel", + "model/credential/login/complete", + "model/catalog/refresh", + "model/tooling/install", + } +) diff --git a/tests/fixtures/operation_result_digest_vectors.json b/tests/fixtures/operation_result_digest_vectors.json new file mode 100644 index 0000000..cc2d59b --- /dev/null +++ b/tests/fixtures/operation_result_digest_vectors.json @@ -0,0 +1,44 @@ +{ + "source": "CP56 OPERATION_RESULT/1.0.0 formula; independent of J helper", + "vectors": [ + { + "value": {}, + "digest": "bca32c7f85e8328c9805776a42733883e04f0f358c2b97ac00a9538e2b4f7e9f" + }, + { + "value": { + "revision": 1, + "name": "Changed exactly once" + }, + "digest": "f5659a222b58a5109567afbe95c662e7ec9fb12c2002ea4ca9be9a1c85376785" + }, + { + "value": { + "z": "한국어 · English", + "a": [ + null, + false, + 7 + ] + }, + "digest": "d312b684a88236c57206e1e3a43a2ffd69c33845935a464fc71498032bbb07c6" + }, + { + "value": { + "nested": { + "b": "2.5", + "a": "value" + }, + "empty": [] + }, + "digest": "1a59a9351dc1d90ec328496bb7354172bb4ad58a8ee46382cf3df59fa6ea4c82" + }, + { + "value": { + "state": "SUCCEEDED", + "unchanged": true + }, + "digest": "a57f47caee392c540518a8c1705345305f5e60a74d23bb6fbdaccd73a376386f" + } + ] +} diff --git a/tests/fixtures/r2_closed_loop_golden.json b/tests/fixtures/r2_closed_loop_golden.json index fb73be1..abe4ad3 100644 --- a/tests/fixtures/r2_closed_loop_golden.json +++ b/tests/fixtures/r2_closed_loop_golden.json @@ -4122,6 +4122,10 @@ "PreparedResearchExecution", "SandboxExecutionBundle" ], + [ + "L", + "transaction" + ], [ "X", "reject_observation", @@ -7475,11 +7479,16 @@ "_measurement_assessments", "tuple:d01bf921a2fd20c7" ], + [ + "L", + "transaction" + ], [ "X", "accepting_observation", "SandboxRunSpec:92bdabb537c574c0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -13069,7 +13078,8 @@ "X", "accepting_observation", "SandboxRunSpec:92bdabb537c574c0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -13576,6 +13586,10 @@ "project:a04:allowed", "d0a6cfbed30739d9a4438f597479803ad7436faf747e04fb2135ecd2662cb0eb" ], + [ + "L", + "transaction" + ], [ "X", "reject_observation", @@ -17276,7 +17290,8 @@ "X", "accepting_observation", "SandboxRunSpec:92bdabb537c574c0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -23143,7 +23158,8 @@ "X", "accepting_observation", "SandboxRunSpec:39352fb13a8e9978", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -27229,7 +27245,8 @@ "X", "accepting_observation", "SandboxRunSpec:60e264ea04d64b51", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -30951,7 +30968,8 @@ "X", "accepting_observation", "SandboxRunSpec:647dffad191532ca", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -32223,7 +32241,8 @@ "X", "accepting_observation", "SandboxRunSpec:ae549f5c7a2d01bf", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -32525,7 +32544,8 @@ "X", "accepting_observation", "SandboxRunSpec:87ac021e9c054bf0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -33785,7 +33805,8 @@ "X", "accepting_observation", "SandboxRunSpec:8481b5ba1239c926", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -35045,7 +35066,8 @@ "X", "accepting_observation", "SandboxRunSpec:b9f2d962054617b7", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -36518,7 +36540,8 @@ "X", "accepting_observation", "SandboxRunSpec:0848943b2e1ca6a2", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -36958,7 +36981,8 @@ "X", "accepting_observation", "SandboxRunSpec:59a09a8fd8a8c946", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -39468,6 +39492,10 @@ "read_heads", "project:a04:allowed" ], + [ + "L", + "transaction" + ], [ "S", "enter", @@ -40859,7 +40887,8 @@ "X", "accepting_observation", "SandboxRunSpec:5e5ec03e176405ed", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", diff --git a/tests/fixtures/reference_service_golden.json b/tests/fixtures/reference_service_golden.json index 3ddc553..be9703e 100644 --- a/tests/fixtures/reference_service_golden.json +++ b/tests/fixtures/reference_service_golden.json @@ -7,9 +7,10 @@ "error": { "code": -32030, "data": { - "reason_code": "REFERENCE_ANSWER_REVISION_CONFLICT" + "reason_code": "REFERENCE_ANSWER_REVISION_CONFLICT", + "research_saved": true }, - "message": "REFERENCE_ANSWER_REVISION_CONFLICT" + "message": "\uc5f0\uad6c \uacb0\uacfc\ub294 \uc800\uc7a5\ub410\uc2b5\ub2c8\ub2e4. \uae30\uc900(reference) \ub2e8\uacc4\ub9cc \uc2e4\ud328\ud588\uc2b5\ub2c8\ub2e4 (\uc0ac\uc720: REFERENCE_ANSWER_REVISION_CONFLICT). \uacb0\uacfc\ub294 \uae30\ub85d\uc5d0\uc11c \ubcfc \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uac19\uc740 \uc9c8\ubb38\uc744 \uc0c8\ub85c \ubcf4\ub0b4\uba74 \uc5f0\uad6c\uac00 \ub2e4\uc2dc \uc2e4\ud589\ub420 \uc218 \uc788\uc73c\ub2c8 \uba3c\uc800 \uc800\uc7a5\ub41c \uacb0\uacfc\ub97c \ud655\uc778\ud558\uc138\uc694. / The research result was saved. Only the reference step failed (reason: REFERENCE_ANSWER_REVISION_CONFLICT). You can read the result in the history. Sending the same question again as a new request may run the research again, so check the saved result first." } } ] diff --git a/tests/fixtures/supplementable_methods.json b/tests/fixtures/supplementable_methods.json new file mode 100644 index 0000000..95fcfb8 --- /dev/null +++ b/tests/fixtures/supplementable_methods.json @@ -0,0 +1,341 @@ +{ + "schema_version": "1.0.0", + "source_head": "70e6739f08c14a3f54f0ec43adfd46b487fcf4b1", + "methods": [ + "project/activate", + "project/archive", + "project/create", + "project/cutoff/update", + "project/metadata/update", + "project/overlay/update", + "project/policy/update", + "project/reference/import", + "project/restore", + "project/role/assign", + "project/role/revoke", + "revision/baseline/prepare", + "revision/branch/create", + "revision/changeSet/create", + "revision/changeSet/validate", + "revision/merge/propose", + "revision/merge/resolve", + "revision/propose", + "revision/recompute/request", + "revision/restore/propose" + ], + "result_key_shapes": { + "project/activate": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/archive": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/create": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/cutoff/update": [ + [ + "cutoff_at", + "description", + "impact", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/metadata/update": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/overlay/update": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/policy/update": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/reference/import": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "reference", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/restore": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/role/assign": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "role", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/role/revoke": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "role", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "revision/baseline/prepare": [ + [ + "baseline_candidate", + "impact", + "protected_authority_package" + ] + ], + "revision/branch/create": [ + [ + "branch", + "content_mutated" + ] + ], + "revision/changeSet/create": [ + [ + "change_set", + "conflict_state", + "missing_dependencies" + ] + ], + "revision/changeSet/validate": [ + [ + "change_set", + "checks", + "impact_propagation_plan", + "state", + "validation_bundle_digest" + ], + [ + "change_set", + "impact_propagation_plan", + "state", + "validation_bundle_digest" + ] + ], + "revision/merge/propose": [ + [ + "conflict", + "head_mutated", + "merge_proposal", + "receipt", + "semantic_merge" + ] + ], + "revision/merge/resolve": [ + [ + "candidate_revision", + "change_set", + "head_mutated", + "merge_proposal" + ] + ], + "revision/propose": [ + [ + "impact_preview", + "proposal", + "staged_snapshot", + "validation_state" + ] + ], + "revision/recompute/request": [ + [ + "recompute_task" + ] + ], + "revision/restore/propose": [ + [ + "change_set", + "external_effect_rollback_claimed", + "impact_plan", + "restore_proposal" + ] + ] + }, + "evidence": { + "path": "outputs/refactor-vj-20261019/method-evidence.json", + "sha256": "28f24982437dbab8f8fa637e066e10fc730ef3da1af9822e51df4ab69c1374eb" + }, + "scope": "LOCAL; only measured saved result shapes" +} diff --git a/tests/integration/journal_completion_helpers.py b/tests/integration/journal_completion_helpers.py new file mode 100644 index 0000000..576b8b7 --- /dev/null +++ b/tests/integration/journal_completion_helpers.py @@ -0,0 +1,81 @@ +"""Real LOCAL setup with a committed result and a deliberately absent success tail.""" + +from dataclasses import dataclass +from pathlib import Path + +import pytest +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.apps.runtime import AppRuntime +from thoth.domain.deployment_mode import DeploymentMode +from thoth.domain.event import Checkpoint +from thoth.domain.operation import OperationRecord +from thoth.protocol.bus import DispatchTicket +from thoth.protocol.jsonrpc import JsonRpcRequest + + +@dataclass +class Gap: + runtime: AppRuntime + command: JsonRpcRequest + operation: OperationRecord + model: NoModels + + +class TailInterrupted(RuntimeError): + pass + + +async def prepare_gap( + workspace: Path, + monkeypatch: pytest.MonkeyPatch, + *, + mode: DeploymentMode = DeploymentMode.LOCAL, +) -> Gap: + model = NoModels() + runtime = create_runtime(workspace, model_resolver=model, deployment_mode=mode) + try: + value( + await runtime.bus.dispatch( + request( + "project/create", + "create", + { + "project_id": "project:j", + "name": "Original", + "cutoff_at": "2026-10-01T00:00:00Z", + }, + ) + ) + ) + command = request( + "project/metadata/update", + "gap", + { + "project_id": "project:j", + "expected_revision": 0, + "name": "Saved once", + }, + ) + ticket = runtime.bus.claim(command) + assert isinstance(ticket, DispatchTicket) + original = SqliteEventStore.insert_checkpoint + + def fail(store: SqliteEventStore, point: Checkpoint) -> None: + if point.operation_id == ticket.operation.operation_id: + raise TailInterrupted("checkpoint before insert") + original(store, point) + + with monkeypatch.context() as patch: + patch.setattr(SqliteEventStore, "insert_checkpoint", fail) + with pytest.raises(TailInterrupted): + await runtime.bus.execute(ticket) + stored = runtime.bus.read_operation(ticket.operation.operation_id) + assert stored is not None and stored.result is not None + return Gap(runtime, command, stored, model) + except BaseException: + runtime.close() + raise diff --git a/tests/integration/p_r2_failure_helpers.py b/tests/integration/p_r2_failure_helpers.py new file mode 100644 index 0000000..3452256 --- /dev/null +++ b/tests/integration/p_r2_failure_helpers.py @@ -0,0 +1,83 @@ +"""Normal-entry R2 fault instrumentation; snapshots use independent connections.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.test_hypothesis_prediction_outcome_cycle import request, value + +from thoth.adapters.storage import transaction as transaction_module +from thoth.application.services.r2_closed_loop import R2ClosedLoopCoordinator +from thoth.apps.runtime import AppRuntime +from thoth.domain.sandbox import SandboxRunSpec + + +def ambient() -> bool: + return vars(transaction_module)["_AMBIENT"].get() is not None + + +def save(name: str, evidence: dict[str, Any]) -> None: + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/refactor-p-20261022" + / os.environ.get("THOTH_P_RUN_LABEL", "p2-direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{name}.json").write_text( + json.dumps(evidence, indent=2, default=str) + "\n", encoding="utf-8" + ) + + +def capture_exit(patch: pytest.MonkeyPatch, runtime: AppRuntime, evidence: dict[str, Any]) -> None: + original = R2ClosedLoopCoordinator.execute_if_required + + async def execute(coordinator: R2ClosedLoopCoordinator, **kwargs: Any) -> Any: + try: + return await original(coordinator, **kwargs) + finally: + # This runs before the outer handler publishes its terminal operation/error. + evidence["after_phase"] = snapshot(runtime.ledger.engine) + evidence["ambient_after_phase"] = ambient() + + patch.setattr(R2ClosedLoopCoordinator, "execute_if_required", execute) + + +async def revise_hypothesis(runtime: AppRuntime, project: str, spec: SandboxRunSpec) -> None: + action = value( + await runtime.bus.dispatch( + request( + "action/read", "p2-action", {"project_id": project, "action_id": spec.action_id} + ) + ) + )["action"] + hypothesis_id = action["hypothesis_refs"][0] + hypothesis = value( + await runtime.bus.dispatch( + request( + "hypothesis/read", + "p2-hypothesis", + {"project_id": project, "hypothesis_id": hypothesis_id}, + ) + ) + )["hypothesis"] + value( + await runtime.bus.dispatch( + request( + "hypothesis/revise", + "p2-concurrent-edit", + { + "project_id": project, + "hypothesis_id": hypothesis_id, + "expected_revision_digest": hypothesis["revision_digest"], + "patch": {"statement": "Operator revised the measurement hypothesis"}, + "evidence_refs": hypothesis["evidence_refs"], + "reason": "new operator evidence", + }, + ) + ) + ) diff --git a/tests/integration/test_journal_completion_concurrency.py b/tests/integration/test_journal_completion_concurrency.py new file mode 100644 index 0000000..ee70add --- /dev/null +++ b/tests/integration/test_journal_completion_concurrency.py @@ -0,0 +1,127 @@ +"""Real SQLite callers coordinate normal tails and repeated repairs across runtimes.""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Event +from time import monotonic + +import pytest +from tests.integration.journal_completion_helpers import TailInterrupted, prepare_gap +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.workspace_owner import workspace_owner_key +from thoth.domain.event import Checkpoint +from thoth.protocol import journal_completion_gate as gates +from thoth.protocol.bus import DispatchTicket + + +async def test_two_runtime_replays_add_only_one_tail( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + second = create_runtime(tmp_path, model_resolver=gap.model) + try: + with ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(asyncio.run, gap.runtime.bus.dispatch(gap.command)) + b = pool.submit(asyncio.run, second.bus.dispatch(gap.command)) + left, right = a.result(timeout=10), b.result(timeout=10) + assert left == right and value(left) == gap.operation.result + events = SqliteEventStore(gap.runtime.ledger.engine) + assert [e.event_type for e in events.list_events(gap.operation.operation_id)] == [ + "operation.started", + "project/metadata/updated", + "operation.succeeded", + ] + assert len(events.list_checkpoints(gap.operation.operation_id)) == 1 + assert gap.model.calls == 0 + assert (workspace_owner_key(tmp_path), gap.operation.operation_id) not in vars(gates)[ + "_entries" + ] + finally: + second.close() + gap.runtime.close() + + +@pytest.mark.parametrize("normal_fails", [False, True]) +async def test_replay_waits_for_original_tail_before_deciding_what_is_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, normal_fails: bool +) -> None: + model = NoModels() + first = create_runtime(tmp_path, model_resolver=model) + second = create_runtime(tmp_path, model_resolver=model) + waiting, release = Event(), Event() + try: + value( + await first.bus.dispatch( + request( + "project/create", + "create", + { + "project_id": "project:j", + "name": "Original", + "cutoff_at": "2026-10-01T00:00:00Z", + }, + ) + ) + ) + command = request( + "project/metadata/update", + "race", + { + "project_id": "project:j", + "expected_revision": 0, + "name": "Saved once", + }, + ) + ticket = first.bus.claim(command) + assert isinstance(ticket, DispatchTicket) + identifier = ticket.operation.operation_id + original = SqliteEventStore.insert_checkpoint + + def paused(self: SqliteEventStore, point: Checkpoint) -> None: + if point.operation_id == identifier and self._engine is first.ledger.engine: # pyright: ignore[reportPrivateUsage] + waiting.set() + assert release.wait(timeout=10) + if normal_fails: + raise TailInterrupted("original tail interrupted") + original(self, point) + + monkeypatch.setattr(SqliteEventStore, "insert_checkpoint", paused) + with ThreadPoolExecutor(max_workers=2) as pool: + writer = pool.submit(asyncio.run, first.bus.execute(ticket)) + try: + assert waiting.wait(timeout=10) + replay = pool.submit(asyncio.run, second.bus.dispatch(command)) + key = (workspace_owner_key(tmp_path), identifier) + deadline = monotonic() + 10 + while vars(gates)["_entries"][key].users < 2 and monotonic() < deadline: + Event().wait(0.005) + assert vars(gates)["_entries"][key].users == 2 + assert not replay.done() + release.set() + if normal_fails: + with pytest.raises(TailInterrupted): + writer.result(timeout=10) + else: + assert writer.result(timeout=10) == replay.result(timeout=10) + assert value(replay.result(timeout=10))["revision"] == 1 + finally: + release.set() + store = SqliteEventStore(second.ledger.engine) + events = store.list_events(identifier) + assert [e.event_type for e in events] == [ + "operation.started", + "project/metadata/updated", + "operation.succeeded", + ] + assert len(store.list_checkpoints(identifier)) == 1 + assert all((e.payload.get("supplemented") is True) is normal_fails for e in events[1:]) + assert key not in vars(gates)["_entries"] and model.calls == 0 + finally: + release.set() + second.close() + first.close() diff --git a/tests/integration/test_journal_completion_faults.py b/tests/integration/test_journal_completion_faults.py new file mode 100644 index 0000000..9652753 --- /dev/null +++ b/tests/integration/test_journal_completion_faults.py @@ -0,0 +1,143 @@ +"""Real repair transactions defer infrastructure failures but preserve authority failures.""" + +from contextlib import nullcontext +from datetime import datetime, timedelta +from pathlib import Path +from typing import cast + +import pytest +from sqlalchemy.exc import OperationalError +from tests.atomicity.harness import snapshot +from tests.integration.journal_completion_helpers import prepare_gap +from tests.integration.storage_coverage_helpers import value + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.application.services.operation_journal import OperationJournal +from thoth.application.services.resource_scope_service import ResourceScopeService +from thoth.domain.deployment_mode import DeploymentMode +from thoth.domain.event import EventRecord +from thoth.domain.operation import OperationRecord +from thoth.domain.resource_scope import ResourceScopeError +from thoth.protocol.journal_completion import JournalCompletion + + +async def test_operational_write_failure_rolls_back_and_preserves_success( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + original = SqliteEventStore.insert_event + try: + before = snapshot(gap.runtime.ledger.engine) + + def failed(self: SqliteEventStore, event: EventRecord) -> None: + original(self, event) + if event.operation_id == gap.operation.operation_id: + raise OperationalError("synthetic journal I/O", {}, OSError("synthetic")) + + with monkeypatch.context() as patch: + patch.setattr(SqliteEventStore, "insert_event", failed) + response = await gap.runtime.bus.dispatch(gap.command) + assert value(response) == gap.operation.result + assert snapshot(gap.runtime.ledger.engine) == before + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + events = SqliteEventStore(gap.runtime.ledger.engine) + assert len(events.list_events(gap.operation.operation_id)) == 3 + assert len(events.list_checkpoints(gap.operation.operation_id)) == 1 + assert gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_replay_authority_denial_happens_before_any_journal_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + before = snapshot(gap.runtime.ledger.engine) + + def denied(self: ResourceScopeService, operation: OperationRecord) -> None: + assert operation.operation_id == gap.operation.operation_id + raise ResourceScopeError("RESOURCE_SCOPE_READ_DENIED") + + monkeypatch.setattr(ResourceScopeService, "require_operation", denied) + response = await gap.runtime.bus.dispatch(gap.command) + assert response.error is not None + assert response.error.data["reason_code"] == "RESOURCE_SCOPE_READ_DENIED" + assert snapshot(gap.runtime.ledger.engine) == before and gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_corrupt_result_binding_is_not_hidden_by_repair( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + with gap.runtime.ledger.engine.begin() as connection: + connection.exec_driver_sql( + "UPDATE operations SET result_json=? WHERE operation_id=?", + ('{"tampered":true}', gap.operation.operation_id), + ) + before = snapshot(gap.runtime.ledger.engine) + response = await gap.runtime.bus.dispatch(gap.command) + assert response.error is not None + assert response.error.data["reason_code"] == "OPERATION_RESOURCE_BINDING_INVALID" + assert snapshot(gap.runtime.ledger.engine) == before and gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_missing_workspace_pin_defers_without_acquiring_a_new_lock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + completion = cast(JournalCompletion, vars(gap.runtime.bus)["_journal_completion"]) + monkeypatch.setattr(completion, "_pin", lambda: nullcontext(False)) + before = snapshot(gap.runtime.ledger.engine) + assert completion.attempt(gap.operation).reason == "WORKSPACE_NOT_OWNED" + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + assert snapshot(gap.runtime.ledger.engine) == before + finally: + gap.runtime.close() + + +async def test_hosted_replay_does_not_supplement( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch, mode=DeploymentMode.HOSTED_REVIEW) + try: + assert vars(gap.runtime.bus)["_journal_completion"] is None + before = snapshot(gap.runtime.ledger.engine) + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + assert snapshot(gap.runtime.ledger.engine) == before and gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_sqlite_repair_rolls_back_clock_order_conflict( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + store = SqliteEventStore(gap.runtime.ledger.engine) + old_time = store.list_events(gap.operation.operation_id)[0].created_at - timedelta( + seconds=1 + ) + + class BackwardClock: + def now(self) -> datetime: + return old_time + + completion = cast(JournalCompletion, vars(gap.runtime.bus)["_journal_completion"]) + journal = cast(OperationJournal, vars(gap.runtime.bus)["_journal"]) + before = snapshot(gap.runtime.ledger.engine) + with monkeypatch.context() as patch: + patch.setattr(journal, "_clock", BackwardClock()) + outcome = completion.attempt(gap.operation) + assert outcome.status == "DEFERRED" and outcome.reason == "UNSAFE_EVENT_CHAIN_OR_ORDER" + assert snapshot(gap.runtime.ledger.engine) == before + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + assert len(store.list_events(gap.operation.operation_id)) == 3 + finally: + gap.runtime.close() diff --git a/tests/integration/test_journal_completion_process.py b/tests/integration/test_journal_completion_process.py new file mode 100644 index 0000000..87a99eb --- /dev/null +++ b/tests/integration/test_journal_completion_process.py @@ -0,0 +1,145 @@ +"""Run only after the concurrent FULL has ended: real process termination boundaries.""" + +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels +from tests.integration.test_workspace_runtime_exclusivity_baseline import ( + _child_env, # pyright: ignore[reportPrivateUsage] +) + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.workspace_owner import WorkspaceInUse +from thoth.apps.runtime import AppRuntime + +ROOT = Path(__file__).resolve().parents[2] +CHILD = r""" +import asyncio, sys, time +from pathlib import Path +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request,value +from tests.integration.test_operation_journal_failure_boundaries import NoModels +from thoth.adapters.storage.events import SqliteEventStore + +async def main(workspace, marker, phase): + model=NoModels() + runtime=create_runtime(workspace,model_resolver=model) + value(await runtime.bus.dispatch(request('project/create','create',{ + 'project_id':'project:j-process','name':'Original','cutoff_at':'2026-10-01T00:00:00Z'}))) + command=request('project/metadata/update','gap',{ + 'project_id':'project:j-process','expected_revision':0,'name':'Saved once'}) + ticket=runtime.bus.claim(command) + identifier=ticket.operation.operation_id + point_insert=SqliteEventStore.insert_checkpoint + event_insert=SqliteEventStore.insert_event + def ready(): + assert model.calls==0 + marker.write_text('ready',encoding='utf-8') + time.sleep(120) + raise AssertionError('parent did not terminate the test child') + if phase=='repair-checkpoint': + def initial_gap(self,point): + if point.operation_id==identifier:raise RuntimeError('initial gap') + point_insert(self,point) + SqliteEventStore.insert_checkpoint=initial_gap + try:await runtime.bus.execute(ticket) + except RuntimeError as exc:assert str(exc)=='initial gap' + else:raise AssertionError('initial gap not reached') + def checkpoint(self,point): + if point.operation_id==identifier and phase=='normal-checkpoint':ready() + point_insert(self,point) + if point.operation_id==identifier and phase=='repair-checkpoint':ready() + def event(self,item): + event_insert(self,item) + if (item.operation_id==identifier + and item.event_type=='project/metadata/updated' + and phase=='normal-notification'): + ready() + SqliteEventStore.insert_checkpoint=checkpoint + SqliteEventStore.insert_event=event + if phase=='repair-checkpoint':await runtime.bus.dispatch(command) + else:await runtime.bus.execute(ticket) + raise AssertionError('termination point not reached') +asyncio.run(main(Path(sys.argv[1]),Path(sys.argv[2]),sys.argv[3])) +""" + + +def _open(workspace: Path, model: NoModels) -> AppRuntime: + """Use the existing bounded Windows lock-release retry from the owner recovery tests.""" + deadline = time.monotonic() + 10 + while True: + try: + return create_runtime(workspace, model_resolver=model) + except WorkspaceInUse: + if time.monotonic() > deadline: + raise + time.sleep(0.02) + + +@pytest.mark.parametrize("phase", ["normal-checkpoint", "normal-notification", "repair-checkpoint"]) +async def test_process_exit_preserves_result_and_atomic_supplement( + tmp_path: Path, phase: str +) -> None: + workspace, marker, home = tmp_path / "workspace", tmp_path / "ready", tmp_path / "home" + home.mkdir() + log = tmp_path / "child.log" + with log.open("wb") as sink: + child = subprocess.Popen( + [sys.executable, "-B", "-c", CHILD, str(workspace), str(marker), phase], + cwd=ROOT, + env=_child_env(home), + stdout=sink, + stderr=sink, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + try: + deadline = time.monotonic() + 60 + while not marker.exists() and time.monotonic() < deadline: + assert child.poll() is None, log.read_text(errors="replace")[-2000:] + time.sleep(0.025) + assert marker.exists(), "child did not reach the requested journal boundary" + child.kill() + child.wait(timeout=10) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=10) + model = NoModels() + runtime = _open(workspace, model) + try: + with runtime.ledger.engine.connect() as connection: + row = connection.exec_driver_sql( + "SELECT operation_id,state FROM operations WHERE idempotency_key='gap'" + ).one() + identifier = str(row[0]) + assert row[1] == "SUCCEEDED" + journal = SqliteEventStore(runtime.ledger.engine) + assert len(journal.list_checkpoints(identifier)) == ( + 1 if phase == "normal-notification" else 0 + ) + assert len(journal.list_events(identifier)) == (2 if phase == "normal-notification" else 1) + before = snapshot(runtime.ledger.engine) + command = request( + "project/metadata/update", + "gap", + { + "project_id": "project:j-process", + "expected_revision": 0, + "name": "Saved once", + }, + ) + assert value(await runtime.bus.dispatch(command))["revision"] == 1 + after = snapshot(runtime.ledger.engine) + assert all(before[t] == after[t] for t in before if t not in {"events", "checkpoints"}) + assert len(journal.list_events(identifier)) == 3 + assert len(journal.list_checkpoints(identifier)) == 1 + assert value(await runtime.bus.dispatch(command))["revision"] == 1 + assert snapshot(runtime.ledger.engine) == after and model.calls == 0 + finally: + runtime.close() diff --git a/tests/integration/test_journal_completion_replay.py b/tests/integration/test_journal_completion_replay.py new file mode 100644 index 0000000..44af230 --- /dev/null +++ b/tests/integration/test_journal_completion_replay.py @@ -0,0 +1,173 @@ +"""Measured LOCAL success gaps are repaired once without rerunning their effects.""" + +from pathlib import Path + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.operations import SqliteOperationStore +from thoth.domain.event import Checkpoint, EventRecord +from thoth.protocol.bus import DispatchTicket + +NOTIFICATIONS = { + "project/metadata/update": ["project/metadata/updated"], + "project/archive": ["project/archived", "project/statusChanged"], + "project/restore": ["project/statusChanged"], +} + + +class InjectedTailFailure(RuntimeError): + pass + + +@pytest.mark.parametrize( + ("method", "gap"), + [ + *( + ("project/metadata/update", gap) + for gap in [ + "checkpoint-before", + "checkpoint-after", + "notification-before", + "notification-after", + "success-before", + "success-after", + ] + ), + ("project/archive", "notification-before"), + ("project/archive", "second-notification-before"), + ("project/archive", "second-notification-after"), + ("project/restore", "notification-before"), + ], +) +async def test_gap_repair_preserves_result_and_existing_rows( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: str, gap: str +) -> None: + model = NoModels() + runtime = create_runtime(tmp_path, model_resolver=model) + command_fields: dict[str, object] = {"project_id": "project:j", "expected_revision": 0} + try: + value( + await runtime.bus.dispatch( + request( + "project/create", + "create", + { + "project_id": "project:j", + "name": "Original", + "cutoff_at": "2026-10-01T00:00:00Z", + }, + ) + ) + ) + if method == "project/metadata/update": + command_fields["name"] = "Saved once" + if method == "project/restore": + value( + await runtime.bus.dispatch( + request("project/archive", "prepare-archive", command_fields) + ) + ) + command_fields["expected_revision"] = 1 + command = request(method, "gap", command_fields) + ticket = runtime.bus.claim(command) + assert isinstance(ticket, DispatchTicket) + identifier = ticket.operation.operation_id + checkpoint_insert = SqliteEventStore.insert_checkpoint + event_insert = SqliteEventStore.insert_event + hits: list[str] = [] + + def checkpoint(self: SqliteEventStore, point: Checkpoint) -> None: + hit = point.operation_id == identifier and gap.startswith("checkpoint-") + if hit and gap.endswith("before"): + hits.append(gap) + raise InjectedTailFailure(gap) + checkpoint_insert(self, point) + if hit: + hits.append(gap) + raise InjectedTailFailure(gap) + + def event(self: SqliteEventStore, item: EventRecord) -> None: + index = 1 if gap.startswith("second-notification-") else 0 + selected = ( + "operation.succeeded" + if gap.startswith("success-") + else NOTIFICATIONS[method][index] + ) + hit = ( + item.operation_id == identifier + and item.event_type == selected + and not gap.startswith("checkpoint-") + ) + if hit and gap.endswith("before"): + hits.append(gap) + raise InjectedTailFailure(gap) + event_insert(self, item) + if hit: + hits.append(gap) + raise InjectedTailFailure(gap) + + with monkeypatch.context() as patch: + patch.setattr(SqliteEventStore, "insert_checkpoint", checkpoint) + patch.setattr(SqliteEventStore, "insert_event", event) + with pytest.raises(InjectedTailFailure): + await runtime.bus.execute(ticket) + assert hits == [gap] + store = SqliteOperationStore(runtime.ledger.engine) + original = store.read(identifier) + assert ( + original is not None + and original.result is not None + and original.completed_at is not None + ) + original_bytes = original.model_dump_json() + journal = SqliteEventStore(runtime.ledger.engine) + old_events = { + item.event_id: item.model_dump_json() for item in journal.list_events(identifier) + } + old_points = { + item.checkpoint_id: item.model_dump_json() + for item in journal.list_checkpoints(identifier) + } + before = snapshot(runtime.ledger.engine) + repaired = await runtime.bus.dispatch(command) + assert value(repaired) == original.result + stored = store.read(identifier) + assert stored is not None and stored.model_dump_json() == original_bytes + after = snapshot(runtime.ledger.engine) + assert all( + before[name] == after[name] for name in before if name not in {"events", "checkpoints"} + ) + events = journal.list_events(identifier) + assert [item.event_type for item in events] == [ + "operation.started", + *NOTIFICATIONS[method], + "operation.succeeded", + ] + points = journal.list_checkpoints(identifier) + assert len(points) == 1 + for item in events: + if item.event_id in old_events: + assert item.model_dump_json() == old_events[item.event_id] + else: + assert item.payload["supplemented"] is True + assert item.payload["supplemented_for"] == original.completed_at.isoformat() + for point in points: + if point.checkpoint_id in old_points: + assert point.model_dump_json() == old_points[point.checkpoint_id] + else: + assert point.payload["supplemented"] is True + assert point.payload["supplemented_for"] == original.completed_at.isoformat() + assert await runtime.bus.dispatch(command) == repaired + assert snapshot(runtime.ledger.engine) == after + runtime.close() + runtime = create_runtime(tmp_path, model_resolver=model) + assert await runtime.bus.dispatch(command) == repaired + assert snapshot(runtime.ledger.engine) == after + assert model.calls == 0 + finally: + runtime.close() diff --git a/tests/integration/test_m3_hypothesis_first_write_reachability.py b/tests/integration/test_m3_hypothesis_first_write_reachability.py index 64d6bcb..fa284ff 100644 --- a/tests/integration/test_m3_hypothesis_first_write_reachability.py +++ b/tests/integration/test_m3_hypothesis_first_write_reachability.py @@ -178,9 +178,13 @@ def observe_window(assessment: Any, validation: str) -> None: save(case, evidence) @contextmanager - def accept(service: Any, spec: Any, bundle: Any) -> Generator[Any]: + def accept( + service: Any, spec: Any, bundle: Any, *, defer_rejection: bool = False + ) -> Generator[Any]: if not active(): - with originals["accept"](service, spec, bundle) as artifact: + with originals["accept"]( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: yield artifact return evidence["ambient_at_entry"] = vars(transaction_module)["_AMBIENT"].get() is not None @@ -190,7 +194,9 @@ def accept(service: Any, spec: Any, bundle: Any) -> Generator[Any]: evidence["before_phase"] = snapshot(runtime.ledger.engine) evidence["trace"].append("before_original_accept") try: - with originals["accept"](service, spec, bundle) as artifact: + with originals["accept"]( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: evidence["trace"].append("original_accept_entered") yield artifact except BaseException as exc: diff --git a/tests/integration/test_m3_substantive_appraisal_atomicity.py b/tests/integration/test_m3_substantive_appraisal_atomicity.py index b8f5533..ccb4f0c 100644 --- a/tests/integration/test_m3_substantive_appraisal_atomicity.py +++ b/tests/integration/test_m3_substantive_appraisal_atomicity.py @@ -81,9 +81,13 @@ def active() -> bool: ) == ("thread/input", key, project) @contextmanager - def observe_accept(service: SandboxService, spec: Any, bundle: Any) -> Generator[Any]: + def observe_accept( + service: SandboxService, spec: Any, bundle: Any, *, defer_rejection: bool = False + ) -> Generator[Any]: if not active() or spec.project_id != project: - with original_accept(service, spec, bundle) as artifact: + with original_accept( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: yield artifact return phase_attempt.append(spec.attempt_id) @@ -99,7 +103,9 @@ def observe_accept(service: SandboxService, spec: Any, bundle: Any) -> Generator if not evidence["ambient_at_entry"]: evidence["before"] = snapshot(runtime.ledger.engine) try: - with original_accept(service, spec, bundle) as artifact: + with original_accept( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: evidence["trace"].append("original-entered") yield artifact except BaseException as exc: diff --git a/tests/integration/test_m4_memory_control_multiwrite_observation.py b/tests/integration/test_m4_memory_control_multiwrite_observation.py index 4f2b110..16c3e29 100644 --- a/tests/integration/test_m4_memory_control_multiwrite_observation.py +++ b/tests/integration/test_m4_memory_control_multiwrite_observation.py @@ -1,19 +1,17 @@ -"""Step 0 of 96b: what the second write of a two-write MEMORY control route leaves behind. +"""Two-write MEMORY observations, updated to the approved P1 atomic-pair contract. memory/validate (GATE then CANDIDATE revision), memory/changeSet/propose (MEMORY_ENTRY_CANDIDATE then CHANGE_SET_PROPOSAL) and memory/retire/propose (RETIRE_CANDIDATE then -CHANGE_SET_PROPOSAL) call ControlRecordService.create twice. Each create is its own -SqliteControlRecordStore.append commit, the bus does not wrap a handler in a ledger transaction, and -the handler awaits nothing between the two writes. This file observes the current contract and fixes -nothing; it is test-only. +CHANGE_SET_PROPOSAL) call ControlRecordService.create twice. P1 wraps each pair in the handler's +short ledger transaction; the handler awaits nothing between these writes. The second append of the target request is made to fail with an injected error, either before it -writes ("before": the state is exactly what the first write left) or after it wrote ("after": both -rows are committed but the caller sees an error). Everything else is observed from the real runtime: -the rows, the other tables, the sealed failed operation, a same-key replay, a reopened runtime, a -new-key retry and the reads a later request would make. validate also gets one simulated competing +writes ("before") or after it wrote ("after"). Both now roll back the whole pair. Before P1, these +faults left one or two committed rows; the V-P1 evidence retains that baseline. The real runtime +checks all tables, the failed operation, same-key replay, runtime reopening, new-key retry, +and later reads. validate also gets one simulated competing writer (a second revision of the same record between the read and the append), which is what another -process could do; in one process it cannot happen. +process could previously do; this nested injection does not prove cross-process concurrency safety. Evidence files: outputs/refactor-step96-20261015/step0-multiwrite-*.json. """ @@ -300,7 +298,7 @@ async def test_control_route_success_writes_the_two_rows_in_source_order( @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["before", "after"]) @pytest.mark.parametrize("route", ROUTES) -async def test_second_control_write_failure_leaves_the_first_row_and_a_failed_operation( +async def test_second_control_write_failure_rolls_back_both_rows_and_keeps_failed_operation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, route: str, mode: str ) -> None: workspace = tmp_path / "workspace" @@ -320,13 +318,9 @@ async def test_second_control_write_failure_leaves_the_first_row_and_a_failed_op fault_after = snapshot(runtime.ledger.engine) changes = table_changes(before, fault_after) added = rows_after[len(rows_before) :] - expected_added = 1 if mode == "before" else 2 - assert [item["type"] for item in added] == [t for t, _ in EXPECTED_ORDER[route]][ - :expected_added - ] - assert set(domain_changes(changes, before, fault_after, request(method, key, payload))) == { - "control_records" - } + assert added == [] + assert rows_after == rows_before + assert domain_changes(changes, before, fault_after, request(method, key, payload)) == {} reads_after_fault = await later_reads(runtime, route, candidate_id, "fault") runtime.close() @@ -381,16 +375,12 @@ async def test_second_control_write_failure_leaves_the_first_row_and_a_failed_op ) assert [item["type"] for item in retry_added] == [t for t, _ in EXPECTED_ORDER[route]] if route == "validate": - # Current contract: the first row is a visible VALIDATED gate while the candidate - # has not advanced ("before"), or the candidate advanced although the caller saw an - # error ("after"); a new-key retry then adds a second gate with the same digest. - assert reads_after_fault["candidate_state"] == ( - "CANDIDATE" if mode == "before" else "VALIDATED" - ) - assert len(reads_after_fault["gates"]) == 1 - assert reads_after_fault["gates"][0]["state"] == "VALIDATED" + # Approved P1 contract: neither gate nor candidate revision leaks on either fault. + assert reads_after_fault == reads_before + assert reads_after_fault["candidate_state"] == "CANDIDATE" + assert reads_after_fault["gates"] == [] assert reads_after_retry["candidate_state"] == "VALIDATED" - assert len(reads_after_retry["gates"]) == 2 + assert len(reads_after_retry["gates"]) == 1 assert len({gate["gate_digest"] for gate in reads_after_retry["gates"]}) == 1 else: assert reads_after_fault == reads_before diff --git a/tests/integration/test_operation_journal_failure_boundaries.py b/tests/integration/test_operation_journal_failure_boundaries.py index 3758168..52060f1 100644 --- a/tests/integration/test_operation_journal_failure_boundaries.py +++ b/tests/integration/test_operation_journal_failure_boundaries.py @@ -284,6 +284,7 @@ def fail_checkpoint(store: SqliteEventStore, checkpoint: Checkpoint) -> None: "journal_after_failure": journal_status(after, identifier), "journal_after_reopen_retry": journal_status(final, identifier), "no_writes_on_retries": after == retried == final, + "no_writes_after_completion": retried == final, "snapshots": { "before": snapshot_summary(before), "after": snapshot_summary(after), @@ -292,8 +293,8 @@ def fail_checkpoint(store: SqliteEventStore, checkpoint: Checkpoint) -> None: }, "changed_tables": [name for name in before if before[name] != after[name]], "journal_disposition": ( - "OBSERVED_POST_COMPLETION_GAP" - if mode == "checkpoint" and journal_status(final, identifier)["missing"] + "SUPPLEMENTED_POST_COMPLETION_GAP" + if mode == "checkpoint" and not journal_status(final, identifier)["missing"] else "OBSERVED" ), } @@ -312,7 +313,26 @@ def fail_checkpoint(store: SqliteEventStore, checkpoint: Checkpoint) -> None: f"running_tasks={live_tasks, reopened_tasks}" ) else: - assert after == retried == final + if mode == "checkpoint": + assert journal_status(after, identifier)["missing"] + assert journal_status(final, identifier)["missing"] == [] + assert retried == final + assert all( + after[name] == final[name] + for name in after + if name not in {"events", "checkpoints"} + ) + added_events = [json.loads(row) for row in set(final["events"]) - set(after["events"])] + added_points = [ + json.loads(row) for row in set(final["checkpoints"]) - set(after["checkpoints"]) + ] + assert len(added_events) == 2 and len(added_points) == 1 + for row in (*added_events, *added_points): + payload = json.loads(row["payload_json"]) + assert payload["supplemented"] is True + assert payload["supplemented_for"] == operation["completed_at"] + else: + assert after == retried == final assert same_runtime.result == after_reopen.result stored_result = operation["result_json"] assert isinstance(stored_result, str) diff --git a/tests/integration/test_p_memory_atomicity.py b/tests/integration/test_p_memory_atomicity.py new file mode 100644 index 0000000..e5ac9e7 --- /dev/null +++ b/tests/integration/test_p_memory_atomicity.py @@ -0,0 +1,150 @@ +"""Real RPC failures at either append must publish neither memory control record.""" + +from pathlib import Path +from typing import Any + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.test_m4_memory_control_multiwrite_observation import ( + EXPECTED_ORDER, + ROUTES, + call, + domain_changes, + later_reads, + prepare, + request, + rows, + table_changes, +) + +from thoth.adapters.storage.control_record import SqliteControlRecordStore +from thoth.application.services.control_record_service import ControlRecordService +from thoth.apps.runtime import create_runtime +from thoth.domain.control_record import ControlRecord +from thoth.protocol.deferred import current_operation + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("write_number", [1, 2]) +@pytest.mark.parametrize("after_write", [False, True]) +async def test_append_fault_rolls_back_pair_and_replay_keeps_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + route: str, + write_number: int, + after_write: bool, +) -> None: + workspace = tmp_path / "workspace" + runtime, method, payload, candidate_id = await prepare(workspace, route) + key = f"p1-{route}-{write_number}-{after_write}" + original = SqliteControlRecordStore.append + hits: list[str] = [] + + def append(store: SqliteControlRecordStore, record: ControlRecord) -> None: + operation = current_operation.get() + if operation is None or operation.idempotency_key != key: + original(store, record) + return + hits.append(record.record_type) + if len(hits) == write_number: + if after_write: + original(store, record) + raise RuntimeError("P1 injected append failure") + original(store, record) + + try: + reads_before = await later_reads(runtime, route, candidate_id, "p1-before") + rows_before = rows(runtime) + before = snapshot(runtime.ledger.engine) + command = request(method, key, payload) + with monkeypatch.context() as patch: + patch.setattr(SqliteControlRecordStore, "append", append) + failed = await runtime.bus.dispatch(command) + assert failed.error is not None + assert hits == [kind for kind, _ in EXPECTED_ORDER[route]][:write_number] + after = snapshot(runtime.ledger.engine) + assert rows(runtime) == rows_before + assert domain_changes(table_changes(before, after), before, after, command) == {} + assert await later_reads(runtime, route, candidate_id, "p1-after") == reads_before + runtime.close() + runtime = create_runtime(workspace) + assert rows(runtime) == rows_before + before_replay = snapshot(runtime.ledger.engine) + replay = await runtime.bus.dispatch(command) + assert replay.error == failed.error + assert snapshot(runtime.ledger.engine) == before_replay + await call(runtime, method, key + "-fresh", payload) + added = rows(runtime)[len(rows_before) :] + assert [row["type"] for row in added] == [kind for kind, _ in EXPECTED_ORDER[route]] + finally: + runtime.close() + + +async def test_bound_helper_looks_up_create_after_handler_construction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, method, payload, _ = await prepare(tmp_path / "workspace", "validate") + key = "p1-create-monkeypatch-after-construction" + original = ControlRecordService.create + hits: list[str] = [] + + def create( + service: ControlRecordService, + *, + project_id: str, + namespace: str, + record_type: str, + state: str, + payload: dict[str, object], + record_id: str | None = None, + ) -> ControlRecord: + operation = current_operation.get() + if operation is not None and operation.idempotency_key == key: + hits.append(record_type) + return original( + service, + project_id=project_id, + namespace=namespace, + record_type=record_type, + state=state, + payload=payload, + record_id=record_id, + ) + + try: + monkeypatch.setattr(ControlRecordService, "create", create) + await call(runtime, method, key, payload) + assert hits == ["GATE", "CANDIDATE"] + finally: + runtime.close() + + +@pytest.mark.parametrize("route", ROUTES) +async def test_partial_first_append_is_not_visible_to_independent_reader( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + runtime, method, payload, _ = await prepare(tmp_path / "workspace", route) + key = f"p1-reader-{route}" + original = SqliteControlRecordStore.append + first_views: list[list[dict[str, Any]]] = [] + seen: list[str] = [] + before = rows(runtime) + + def append(store: SqliteControlRecordStore, record: ControlRecord) -> None: + original(store, record) + operation = current_operation.get() + if operation is not None and operation.idempotency_key == key: + seen.append(record.record_type) + if len(seen) == 1: + # rows() deliberately opens another connection, bypassing ambient UoW. + first_views.append(rows(runtime)) + + try: + with monkeypatch.context() as patch: + patch.setattr(SqliteControlRecordStore, "append", append) + await call(runtime, method, key, payload) + assert seen == [kind for kind, _ in EXPECTED_ORDER[route]] + assert first_views == [before] + assert len(rows(runtime)) == len(before) + 2 + finally: + runtime.close() diff --git a/tests/integration/test_p_r2_failure_atomicity.py b/tests/integration/test_p_r2_failure_atomicity.py new file mode 100644 index 0000000..11bba5a --- /dev/null +++ b/tests/integration/test_p_r2_failure_atomicity.py @@ -0,0 +1,231 @@ +"""Prepared R2 failures publish hold/reject and final completion/candidate atomically.""" + +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path +from typing import Any, cast + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.p_r2_failure_helpers import ambient, capture_exit, revise_hypothesis, save +from tests.integration.test_a02_autonomous_acquisition import StaticModelResolver +from tests.integration.test_a04_r2_closed_loop import A04R2Model +from tests.integration.test_hypothesis_prediction_outcome_cycle import ( + MeasurementSandbox, + prepare_measurement, + request, + value, +) + +from thoth.application.services.r2_closed_loop import R2ClosedLoopCoordinator +from thoth.application.services.r2_test_lifecycle import R2TestLifecycle +from thoth.application.services.sandbox_service import SandboxService +from thoth.apps.runtime import create_runtime +from thoth.domain.sandbox import SandboxErrorCode, SandboxExecutionState, SandboxFailure +from thoth.ports.model import ModelPort + + +@pytest.mark.parametrize( + "path", ["stale", "accept_success", "accept_failure", "assessment", "finalization"] +) +@pytest.mark.parametrize("after_write", [False, True]) +@pytest.mark.parametrize("write_number", [1, 2]) +async def test_prepared_rejection_pair_rolls_back_on_write_fault( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, path: str, after_write: bool, write_number: int +) -> None: + states = (SandboxExecutionState.FAILED,) if path == "accept_failure" else () + runtime, sandbox, project, thread, _ = await prepare_measurement( + tmp_path, monkeypatch, states=states + ) + evidence: dict[str, Any] = {"path": path, "trace": [], "faults": 0, "await_ambient": []} + originals = {"hold": R2TestLifecycle.hold, "reject": SandboxService.reject_observation} + + async def during_run(spec: Any) -> None: + evidence["await_ambient"].append(ambient()) + if path == "stale": + await revise_hypothesis(runtime, project, spec) + + sandbox.on_result = during_run + + def pair_call(name: str, instance: Any, *args: Any, **kwargs: Any) -> Any: + if not evidence["trace"]: + evidence["before_phase"] = snapshot(runtime.ledger.engine) + evidence["trace"].append(name) + target = len(evidence["trace"]) == write_number + if target and not after_write: + evidence["faults"] += 1 + raise RuntimeError("P2 second pair write fault") + result = originals[name](instance, *args, **kwargs) + if target: + evidence["faults"] += 1 + raise RuntimeError("P2 second pair write fault") + return result + + def hold(instance: Any, *args: Any, **kwargs: Any) -> Any: + return pair_call("hold", instance, *args, **kwargs) + + def reject(instance: Any, *args: Any, **kwargs: Any) -> Any: + return pair_call("reject", instance, *args, **kwargs) + + original_basis = SandboxService._validate_admission_basis # pyright: ignore[reportPrivateUsage] + + def basis(instance: Any, *args: Any, **kwargs: Any) -> Any: + if sandbox.seen_specs: + raise SandboxFailure(SandboxErrorCode.INPUT_INVALID, "P2 admission changed") + return original_basis(instance, *args, **kwargs) + + def invalid_assessment(*args: Any, **kwargs: Any) -> Any: + raise ValueError("RESEARCH_TEST_ASSESSMENT_REJECTED") + + def invalid_completion(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError("P2 finalization failed") + + try: + command = request("thread/input", "p2-pair", {"project_id": project, "thread_id": thread}) + with monkeypatch.context() as patch: + capture_exit(patch, runtime, evidence) + patch.setattr(R2TestLifecycle, "hold", hold) + patch.setattr(SandboxService, "reject_observation", reject) + if path.startswith("accept_"): + patch.setattr(SandboxService, "_validate_admission_basis", basis) + elif path == "assessment": + patch.setattr(R2TestLifecycle, "prepare_assessments", invalid_assessment) + elif path == "finalization": + patch.setattr(R2TestLifecycle, "complete", invalid_completion) + response = await runtime.bus.dispatch(command) + evidence["response"] = response.model_dump(mode="json") + save(f"pair-{path}-{write_number}-{after_write}", evidence) + assert evidence["faults"] == 1 + expected = ["hold", "reject"] if path == "stale" else ["reject", "hold"] + assert evidence["trace"] == expected[:write_number] + assert evidence["await_ambient"] == [False] + assert evidence["ambient_after_phase"] is False + assert response.error is not None + assert evidence["after_phase"] == evidence["before_phase"] + terminal = snapshot(runtime.ledger.engine) + replay = await runtime.bus.dispatch(command) + assert replay.error == response.error + assert snapshot(runtime.ledger.engine) == terminal + assert len(sandbox.seen_specs) == 1 + finally: + runtime.close() + + +@pytest.mark.parametrize("state", [SandboxExecutionState.FAILED, SandboxExecutionState.TIMED_OUT]) +@pytest.mark.parametrize("after_candidate", [False, True]) +@pytest.mark.parametrize("sandbox_error", [False, True]) +async def test_final_prepared_completion_and_candidate_roll_back_together( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + state: SandboxExecutionState, + after_candidate: bool, + sandbox_error: bool, +) -> None: + runtime, sandbox, project, thread, _ = await prepare_measurement( + tmp_path, monkeypatch, states=(state,), max_transient_retries=0 + ) + evidence: dict[str, Any] = {"complete_hits": 0, "candidate_hits": 0, "await_ambient": []} + original_accept = SandboxService.accepting_observation + original_complete = R2TestLifecycle.complete + original_failure = R2ClosedLoopCoordinator._failure_execution # pyright: ignore[reportPrivateUsage] + + async def during_run(spec: Any) -> None: + evidence["await_ambient"].append(ambient()) + + sandbox.on_result = during_run + + @contextmanager + def accept(instance: Any, spec: Any, bundle: Any, **kwargs: Any) -> Generator[Any]: + evidence["before_phase"] = snapshot(runtime.ledger.engine) + with original_accept(instance, spec, bundle, **kwargs) as artifact: + yield artifact + + def complete(instance: Any, *args: Any, **kwargs: Any) -> Any: + evidence["complete_hits"] += 1 + return original_complete(instance, *args, **kwargs) + + def failure(instance: Any, **kwargs: Any) -> Any: + evidence["candidate_hits"] += 1 + evidence["has_test_completion"] = kwargs.get("test_completion") is not None + if after_candidate: + result = original_failure(instance, **kwargs) + evidence["candidate_projection"] = result.projection + if sandbox_error: + raise SandboxFailure( + SandboxErrorCode.INPUT_INVALID, "P2 candidate error, not admission" + ) + raise RuntimeError("P2 candidate publication fault") + + try: + command = request("thread/input", "p2-final", {"project_id": project, "thread_id": thread}) + with monkeypatch.context() as patch: + capture_exit(patch, runtime, evidence) + patch.setattr(SandboxService, "accepting_observation", accept) + patch.setattr(R2TestLifecycle, "complete", complete) + patch.setattr(R2ClosedLoopCoordinator, "_failure_execution", failure) + response = await runtime.bus.dispatch(command) + evidence["response"] = response.model_dump(mode="json") + save(f"final-{state.value}-{after_candidate}-{sandbox_error}", evidence) + assert evidence["complete_hits"] == evidence["candidate_hits"] == 1 + assert evidence["has_test_completion"] is True + assert evidence["await_ambient"] == [False] + assert evidence["ambient_after_phase"] is False + assert response.error is not None + assert evidence["after_phase"] == evidence["before_phase"] + terminal = snapshot(runtime.ledger.engine) + replay = await runtime.bus.dispatch(command) + assert replay.error == response.error + assert snapshot(runtime.ledger.engine) == terminal + assert len(sandbox.seen_specs) == 1 + runtime.close() + runtime = create_runtime( + tmp_path / "allowed", + sandbox_adapter=sandbox, + model_resolver=StaticModelResolver(cast(ModelPort, A04R2Model())), + ) + assert snapshot(runtime.ledger.engine) == terminal + replay = await runtime.bus.dispatch(command) + assert replay.error == response.error + assert snapshot(runtime.ledger.engine) == terminal + again = value( + await runtime.bus.dispatch( + request("thread/input", "p2-new-key", {"project_id": project, "thread_id": thread}) + ) + )["r2_closed_loop"] + assert again["reason_code"] == "RESEARCH_TEST_RESULT_RECONCILIATION_REQUIRED" + assert again["pending_result"]["exits"] == ["CLEAR_AND_RERUN"] + assert len(sandbox.seen_specs) == 1 + finally: + runtime.close() + + +async def test_retry_awaits_stay_outside_transactions( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, sandbox, project, thread, _ = await prepare_measurement( + tmp_path, + monkeypatch, + states=(SandboxExecutionState.TIMED_OUT, SandboxExecutionState.SUCCEEDED), + max_transient_retries=1, + ) + original = MeasurementSandbox.run + boundaries: list[tuple[str, bool]] = [] + + async def run(adapter: MeasurementSandbox, spec: Any) -> Any: + boundaries.append(("before", ambient())) + result = await original(adapter, spec) + boundaries.append(("after", ambient())) + return result + + try: + monkeypatch.setattr(MeasurementSandbox, "run", run) + result = value( + await runtime.bus.dispatch( + request("thread/input", "p2-retry", {"project_id": project, "thread_id": thread}) + ) + ) + assert result["r2_closed_loop"]["terminal_state"] == "COMPLETED" + assert boundaries == [("before", False), ("after", False)] * 2 + assert len(sandbox.seen_specs) == 2 + finally: + runtime.close() diff --git a/tests/integration/test_p_reference_failure_notice.py b/tests/integration/test_p_reference_failure_notice.py new file mode 100644 index 0000000..8c2324b --- /dev/null +++ b/tests/integration/test_p_reference_failure_notice.py @@ -0,0 +1,256 @@ +"""A real post-research reference rejection must describe only readable saved research.""" + +import json +import os +from pathlib import Path +from typing import Any + +import pytest +from tests.integration.reference_helpers import reference_harness +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_a02_autonomous_acquisition import DynamicA02Model, StaticModelResolver +from tests.integration.test_research_request_v2 import ControlledResearchModel + +from thoth.application.commands.reference import ReferenceThreadEntry +from thoth.application.commands.thread_analysis import ThreadAnalysisCommandHandlers +from thoth.application.services.revision_service import CommitResult +from thoth.domain.behavior_execution import BehaviorExecutionRecord +from thoth.domain.reference import ReferenceError +from thoth.protocol.deferred import current_operation + + +def save(name: str, evidence: dict[str, Any]) -> None: + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/refactor-p-20261022" + / os.environ.get("THOTH_P_RUN_LABEL", "p3-direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{name}.json").write_text( + json.dumps(evidence, indent=2, default=str) + "\n", encoding="utf-8" + ) + + +def install_models(monkeypatch: pytest.MonkeyPatch, version: int) -> list[str]: + controlled = ControlledResearchModel(one=True) + calls: list[str] = [] + dynamic = DynamicA02Model.structured + structured = ControlledResearchModel.structured + + async def legacy_model(instance: DynamicA02Model, call: Any) -> Any: + calls.append(call.role.value) + return await dynamic(instance, call) + + async def controlled_model(instance: ControlledResearchModel, call: Any) -> Any: + calls.append(call.role.value) + return await structured(instance, call) + + def resolve( + instance: StaticModelResolver, *, provider: str, model: str | None + ) -> ControlledResearchModel: + return controlled + + monkeypatch.setattr(DynamicA02Model, "structured", legacy_model) + monkeypatch.setattr(ControlledResearchModel, "structured", controlled_model) + if version == 2: + monkeypatch.setattr(StaticModelResolver, "resolve", resolve) + return calls + + +@pytest.mark.parametrize("version", [1, 2]) +async def test_real_reference_rejection_reports_saved_research_in_both_contracts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, version: int +) -> None: + async with reference_harness(tmp_path) as h: + calls = install_models(monkeypatch, version) + captured: dict[str, Any] = {} + original = ReferenceThreadEntry._reference # pyright: ignore[reportPrivateUsage] + + async def reference(entry: ReferenceThreadEntry, selected: Any, result: Any) -> Any: + op = current_operation.get() + assert op is not None + captured["operation_id"] = op.operation_id + captured["delegate_result"] = result + return await original(entry, selected, result) + + monkeypatch.setattr(ReferenceThreadEntry, "_reference", reference) + command = request( + "thread/input", + "p3-missing-inquiry", + { + "project_id": h.project, + "thread_id": h.thread, + "instruction": "Compare the saved source measurements.", + **({"contract_version": 2} if version == 2 else {}), + "reference_request": {**h.request, "inquiry_id": "missing:p3-inquiry"}, + }, + ) + response = await h.runtime.bus.dispatch(command) + await h.runtime.bus.drain() + assert captured, response + operation = h.runtime.bus.read_operation(captured["operation_id"]) + assert operation is not None and operation.error is not None + error = operation.error + commit = CommitResult.model_validate(captured["delegate_result"]["commit"]) + assert commit.receipt in h.runtime.ledger.read_receipts(h.project) + assert commit.committed_revision_ids + readable: list[dict[str, Any]] = [] + for identifier in commit.committed_revision_ids: + revision = h.runtime.ledger.read_revision_by_id(h.project, identifier) + assert revision is not None + read = value( + await h.runtime.bus.dispatch( + request( + "revision/read", + "p3-read-" + identifier, + {"project_id": h.project, "revision_digest": revision.revision_digest}, + ) + ) + ) + content = value( + await h.runtime.bus.dispatch( + request( + "revision/content/read", + "p3-content-" + identifier, + {"project_id": h.project, "content_digest": read["content_digest"]}, + ) + ) + ) + readable.append({"revision": read, "content": content}) + current = value( + await h.runtime.bus.dispatch( + request( + "thread/read", + "p3-thread", + {"project_id": h.project, "thread_id": h.thread, "view": "FULL"}, + ) + ) + ) + captured.update(error=error, readable=readable, thread_read=current) + save(f"reference-v{version}", captured) + assert error["code"] == (-32030 if version == 1 else -32603) + data = error["data"] + assert isinstance(data, dict) + assert data["reason_code"] == "REFERENCE_INQUIRY_NOT_FOUND" + assert data.get("research_saved") is True + message = str(error["message"]) + assert "기준(reference) 단계만 실패했습니다" in message + assert "Only the reference step failed" in message + if version == 2: + failure = data["failure"] + assert isinstance(failure, dict) + primary = failure["primary"] + assert isinstance(primary, dict) + assert primary["exception_type"] == "RpcApplicationError" + manifest = current.get("current_result") or current.get("previous_result") + assert manifest is not None and manifest["completion"] != "TERMINAL" + assert "중간 기록" in message + before_replay_calls = len(calls) + assert before_replay_calls > 0 + replay = await h.runtime.bus.dispatch(command) + await h.runtime.bus.drain() + assert replay.error is not None + assert replay.error.model_dump(mode="json") == error + assert len(calls) == before_replay_calls + + +@pytest.mark.parametrize("version", [1, 2]) +@pytest.mark.parametrize("failure", ["before", "delegate", "unproven", "generic"]) +async def test_other_failures_never_claim_saved_reference_research( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, version: int, failure: str +) -> None: + async with reference_harness(tmp_path) as h: + calls = install_models(monkeypatch, version) + captured: dict[str, Any] = {} + original_entry = ReferenceThreadEntry.analyze + original_delegate = ThreadAnalysisCommandHandlers.analyze + + async def entry(owner: ReferenceThreadEntry, payload: Any) -> Any: + operation = current_operation.get() + assert operation is not None + captured["operation_id"] = operation.operation_id + captured["calls_at_reference"] = list(calls) + return await original_entry(owner, payload) + + async def delegate(owner: ThreadAnalysisCommandHandlers, payload: Any) -> Any: + if failure == "delegate": + raise ReferenceError("REFERENCE_DELEGATE_TEST_FAILURE") + result = await original_delegate(owner, payload) + if failure == "unproven": + result = {k: v for k, v in result.items() if k != "commit"} + return result + + async def generic(owner: ReferenceThreadEntry, selected: Any, result: Any) -> Any: + raise RuntimeError("private-test-marker-never-present-in-error") + + monkeypatch.setattr(ReferenceThreadEntry, "analyze", entry) + monkeypatch.setattr(ThreadAnalysisCommandHandlers, "analyze", delegate) + if failure == "generic": + monkeypatch.setattr(ReferenceThreadEntry, "_reference", generic) + selected = {**h.request, "inquiry_id": "missing:p3-inquiry"} + if failure == "before": + selected["expected_revision_digest"] = "0" * 64 + response = await h.input( + "p3-negative", + instruction="Compare the saved source measurements.", + reference_request=selected, + **({"contract_version": 2} if version == 2 else {}), + ) + await h.runtime.bus.drain() + assert captured, response + operation = h.runtime.bus.read_operation(captured["operation_id"]) + assert operation is not None and operation.error is not None + error = operation.error + data = error.get("data") or {} + assert isinstance(data, dict) + assert "research_saved" not in data + assert "저장됐습니다" not in str(error["message"]) + assert "were saved" not in str(error["message"]) + assert "private-test-marker" not in str(error) + if version == 2: + failure_record = data["failure"] + assert isinstance(failure_record, dict) + primary = failure_record["primary"] + assert isinstance(primary, dict) + assert primary["exception_type"] == ( + "RuntimeError" if failure == "generic" else "RpcApplicationError" + ) + if failure in {"before", "delegate"}: + assert calls == captured["calls_at_reference"] + assert calls == ( + ["RESEARCH_PLANNER", "EVIDENCE_RERANKER", "SEMANTIC_REVIEWER", "REVIEW_ADJUDICATOR"] + if version == 2 + else [] + ) + else: + assert calls + save(f"negative-{version}-{failure}", {"error": error, "model_roles": calls}) + + +@pytest.mark.parametrize("result", [{"queued": True}, {"unrelated_result": True}]) +async def test_queued_and_unassessed_results_do_not_enter_reference_notice( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, result: dict[str, bool] +) -> None: + async with reference_harness(tmp_path) as h: + + async def delegate(owner: ThreadAnalysisCommandHandlers, payload: Any) -> Any: + return result + + async def reference(owner: ReferenceThreadEntry, selected: Any, payload: Any) -> Any: + raise AssertionError("reference must not run for queued/unassessed input") + + monkeypatch.setattr(ThreadAnalysisCommandHandlers, "analyze", delegate) + monkeypatch.setattr(ReferenceThreadEntry, "_reference", reference) + response = await h.input("p3-no-reference", reference_request=h.request) + observed = value(response) + if result.get("queued") is True: + assert observed == result + else: + assert observed.keys() - result.keys() == {"behavior_execution"} + behavior = BehaviorExecutionRecord.model_validate(observed["behavior_execution"]) + assert (behavior.project_id, behavior.thread_id, behavior.state) == ( + h.project, + h.thread, + "COMPLETED", + ) + assert {k: v for k, v in observed.items() if k != "behavior_execution"} == result diff --git a/tests/integration/test_storage_clock_currentness.py b/tests/integration/test_storage_clock_currentness.py new file mode 100644 index 0000000..d736ca1 --- /dev/null +++ b/tests/integration/test_storage_clock_currentness.py @@ -0,0 +1,662 @@ +"""F-03b: real store appends select B after A despite a backwards or tied wall clock. + +No application runtime, server or model is constructed. Optional canonical-ledger stores are +exercised in their direct projection mode; this is not a claim that canonical head reads are broken. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import Engine, create_engine, literal_column, select +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value + +from thoth.adapters.runtime import SystemClock +from thoth.adapters.storage.action import SqliteActionStore +from thoth.adapters.storage.control_record import SqliteControlRecordStore +from thoth.adapters.storage.criterion_contract import SqliteCriterionContractStore +from thoth.adapters.storage.decision_object import SqliteDecisionObjectStore +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.evidence_graph import SqliteEvidenceGraphStore +from thoth.adapters.storage.execution import SqliteExecutionStore +from thoth.adapters.storage.hypothesis import SqliteHypothesisStore +from thoth.adapters.storage.outcome import SqliteOutcomeStore +from thoth.adapters.storage.schema import metadata +from thoth.adapters.storage.thread_runtime import SqliteThreadRuntimeStore +from thoth.domain.action_full import ( + ActionPlanRecord, + ActionPortfolioRecord, + ActionRecord, + AuthorizationEnvelopeRecord, +) +from thoth.domain.base import DomainModel +from thoth.domain.control_record import ControlRecord +from thoth.domain.criterion_contract import CriterionContractRecord +from thoth.domain.decision_object_full import ( + DecisionObjectRecord, + ObjectAttentionRecord, + ObjectRelationRecord, +) +from thoth.domain.event import Checkpoint +from thoth.domain.evidence_graph import EvidenceSourceRecord +from thoth.domain.execution_full import PlanExecutionRecord, StepExecutionAttemptRecord +from thoth.domain.hypothesis_full import ( + HypothesisPortfolioRecord, + HypothesisRecord, + HypothesisRelationRecord, +) +from thoth.domain.outcome_full import OutcomeAssessmentRecord, OutcomeSeriesRecord +from thoth.domain.thread_runtime import ThreadCheckpoint +from thoth.ports.model import ModelPort + +STAMP = datetime(2026, 10, 23, 12, tzinfo=UTC) +COMMON: dict[str, object] = { + "project_id": "project:f03b", + "object_id": "object:f03b", + "source_object_id": "object:f03b", + "thread_id": "thread:f03b", + "operation_id": "operation:f03b", + "cycle_id": "cycle:f03b", + "portfolio_id": "portfolio:f03b", + "plan_id": "plan:f03b", + "action_id": "action:f03b", + "authorization_id": "authorization:f03b", + "plan_execution_id": "execution:f03b", + "attempt_id": "attempt:f03b", + "step_id": "step:f03b", + "outcome_series_id": "series:f03b", + "outcome_assessment_id": "assessment:f03b", + "criterion_id": "criterion:f03b", + "hypothesis_id": "hypothesis:f03b", + "source_hypothesis_id": "hypothesis:f03b", + "target_hypothesis_id": "hypothesis:other", + "relation_id": "relation:f03b", + "attention_id": "attention:f03b", + "artifact_id": "artifact:f03b", + "record_id": "record:f03b", + "actor_or_agent_ref": "actor:synthetic", + "created_at": STAMP, + "updated_at": STAMP, + "valid_from": STAMP, + "expires_at": STAMP + timedelta(days=1), + "revision_digest": "1" * 64, + "checkpoint_digest": "1" * 64, + "head_set_digest": "1" * 64, + "resume_token": "1" * 64, + "plan_revision_digest": "1" * 64, + "action_plan_revision_digest": "1" * 64, + "expected_working_head_digest": "1" * 64, + "exact_scope_digest": "1" * 64, + "exact_invocation_digest": "1" * 64, + "environment_digest": "1" * 64, + "comparison_baseline_set_digest": "1" * 64, + "baseline_set_digest": "1" * 64, + "record_digest": "1" * 64, + "source_digest": "1" * 64, + "sha256": "1" * 64, + "revision": 1, + "profile_ref": "profile:synthetic", + "evidence_refs": (), + "required_roles": (), + "payload": {"revision": "A"}, +} + + +@dataclass(frozen=True) +class StoreCase: + table: str + model: type[DomainModel] + store: Callable[[Engine], Any] + append: str + revision_field: str + read: str + read_fields: tuple[str | None, ...] + extra: dict[str, object] = field(default_factory=dict) + latest_list: str | None = None + list_fields: tuple[str, ...] = ("project_id",) + digest_field: str = "revision_digest" + read_last: bool = False + time_field: str = "created_at" + + +CASES = ( + StoreCase( + "action_records", + ActionRecord, + SqliteActionStore, + "add_action", + "action_revision_id", + "read_action", + ("project_id", "action_id", None), + { + "primary_purpose": None, + "proposal_state": "DRAFT", + "specification": {}, + "expected_observation_or_change": {}, + "effect_vector": {}, + "impact_set": {}, + "risk_tier": "R0", + "required_processes": (), + }, + "list_actions", + ), + StoreCase( + "action_portfolios", + ActionPortfolioRecord, + SqliteActionStore, + "add_portfolio", + "portfolio_revision_id", + "read_portfolio", + ("project_id", "portfolio_id", None), + { + "action_refs": (), + "decision_need": "synthetic", + "mandatory_criteria": (), + "enhancing_criteria": (), + }, + "list_portfolios", + ), + StoreCase( + "action_plans", + ActionPlanRecord, + SqliteActionStore, + "add_plan", + "plan_revision_id", + "read_plan", + ("project_id", "plan_id", None), + { + "selected_action_refs": (), + "steps": (), + "dependency_edges": (), + "cumulative_impact": {}, + "required_process_union": (), + "required_role_union": (), + "auto_executable_frontier": (), + "point_of_no_return_steps": (), + }, + "list_plans", + ), + StoreCase( + "action_authorizations", + AuthorizationEnvelopeRecord, + SqliteActionStore, + "add_authorization", + "authorization_revision_id", + "read_authorization", + ("project_id", "authorization_id"), + {"predecessor_output_digests": (), "target_baseline_digests": (), "policy_version": "1"}, + "list_authorizations", + ("project_id", "plan_id"), + ), + StoreCase( + "plan_executions", + PlanExecutionRecord, + SqliteExecutionStore, + "add_execution", + "execution_revision_id", + "read_execution", + ("project_id", "plan_execution_id"), + {"execution_profile_ref": "synthetic"}, + "list_executions", + time_field="updated_at", + ), + StoreCase( + "step_execution_attempts", + StepExecutionAttemptRecord, + SqliteExecutionStore, + "add_attempt", + "attempt_revision_id", + "read_attempt", + ("project_id", "attempt_id"), + { + "attempt_number": 1, + "state": "DISPATCHED", + "input_digests": (), + "target_digests": (), + "idempotency_key": "synthetic", + "delivery_guarantee": "AT_MOST_ONCE", + }, + "list_attempts", + ("project_id", "plan_execution_id"), + ), + StoreCase( + "checkpoints", + Checkpoint, + SqliteEventStore, + "insert_checkpoint", + "checkpoint_id", + "latest_checkpoint", + ("operation_id",), + digest_field="checkpoint_digest", + ), + StoreCase( + "thread_checkpoints", + ThreadCheckpoint, + SqliteThreadRuntimeStore, + "put_checkpoint", + "checkpoint_id", + "list_checkpoints", + ("project_id", "thread_id"), + digest_field="checkpoint_digest", + read_last=True, + ), + StoreCase( + "outcome_series", + OutcomeSeriesRecord, + SqliteOutcomeStore, + "add_series", + "series_revision_id", + "read_series", + ("project_id", "outcome_series_id", None), + {"assessment_windows": (), "phase_states": {}}, + "list_series", + ), + StoreCase( + "outcome_assessments", + OutcomeAssessmentRecord, + SqliteOutcomeStore, + "add_assessment", + "assessment_revision_id", + "read_assessment", + ("project_id", "outcome_assessment_id", None), + { + "assessment_phase": "POST", + "profile_version": 1, + "actual_observation_refs": (), + "comparator_refs": (), + "assumptions": (), + }, + "list_assessments", + ), + StoreCase( + "decision_objects", + DecisionObjectRecord, + SqliteDecisionObjectStore, + "add_object", + "object_revision_id", + "read_object", + ("project_id", "object_id", None), + { + "purpose_statement": "synthetic", + "problem_frame": "synthetic", + "focus_refs": (), + "materialization_trigger": "SYNTHETIC", + "trigger_evidence_refs": (), + "entry_criteria": {}, + "exit_criteria": {}, + }, + "list_objects", + ), + StoreCase( + "object_relations", + ObjectRelationRecord, + SqliteDecisionObjectStore, + "add_relation", + "relation_revision_id", + "read_relation", + ("relation_id",), + { + "relation_type": "RELATED", + "target_ref": "object:other", + "semantic_role": "synthetic", + "authority_state": "INFORMAL", + }, + "list_relations", + ("project_id", "source_object_id"), + ), + StoreCase( + "object_attention", + ObjectAttentionRecord, + SqliteDecisionObjectStore, + "add_attention", + "attention_revision_id", + "read_attention", + ("attention_id",), + {"attention_type": "REVIEW", "risk": "LOW", "reason": "synthetic", "condition_ref": "c"}, + "list_attention", + ("project_id", "object_id"), + ), + StoreCase( + "criterion_contracts", + CriterionContractRecord, + SqliteCriterionContractStore, + "add_contract", + "criterion_revision_id", + "read_contract", + ("project_id", "criterion_id", None), + { + "identity": {"name": "synthetic"}, + "profile_refs": (), + "construct_outcome_definition": "x", + "verification_spec": {}, + "required_evidence": (), + "field_evidence_map": {}, + "field_authority_and_version": {}, + "governance": {}, + }, + "list_contracts", + ), + StoreCase( + "hypothesis_records", + HypothesisRecord, + SqliteHypothesisStore, + "add_hypothesis", + "hypothesis_revision_id", + "read_hypothesis", + ("project_id", "hypothesis_id", None), + { + "statement": "synthetic", + "observed_problem": "synthetic", + "primary_intent": None, + "evidence_basis": "UNKNOWN", + "scope": {}, + }, + "list_hypotheses", + ), + StoreCase( + "hypothesis_portfolios", + HypothesisPortfolioRecord, + SqliteHypothesisStore, + "add_portfolio", + "portfolio_revision_id", + "read_portfolio", + ("project_id", "portfolio_id", None), + {"hypothesis_refs": (), "unknown_reserve": {}}, + "list_portfolios", + ), + StoreCase( + "hypothesis_relations", + HypothesisRelationRecord, + SqliteHypothesisStore, + "add_relation", + "relation_revision_id", + "read_relation", + ("relation_id",), + {"relation_type": "RELATED"}, + "list_relations", + ("project_id", "portfolio_id"), + ), + StoreCase( + "evidence_sources", + EvidenceSourceRecord, + SqliteEvidenceGraphStore, + "add_source", + "source_id", + "read_source_by_artifact", + ("artifact_id",), + { + "connector_ref": "fixture:direct-store", + "uri": "fixture://f03b", + "artifact_type": "TEXT", + "authority_status": "INFORMAL", + "security_class": "INTERNAL", + "cutoff_eligibility": "ELIGIBLE", + "lineage_root_id": "source:root", + }, + digest_field="source_digest", + ), + # The plan classifies this table as safe because it has a monotonic version. Check both APIs. + StoreCase( + "control_records", + ControlRecord, + SqliteControlRecordStore, + "append", + "control_revision_id", + "read", + ("project_id", "namespace", "record_id"), + {"namespace": "MEMORY", "record_type": "CANDIDATE", "version": 1, "state": "CANDIDATE"}, + "list", + ("project_id", "namespace"), + digest_field="record_digest", + ), +) + + +def revisions(case: StoreCase, offset: int) -> tuple[DomainModel, DomainModel]: + values = {key: value for key, value in COMMON.items() if key in case.model.model_fields} + first = case.model.model_validate({**values, **case.extra, case.revision_field: "z:revision-A"}) + second = first.model_dump(mode="python") + second.update( + { + case.revision_field: "a:revision-B", + case.digest_field: "2" * 64, + case.time_field: STAMP + timedelta(seconds=offset), + } + ) + for name in ("revision", "version"): + if isinstance(second.get(name), int): + second[name] = int(str(second[name])) + 1 + if "supersedes_revision_digest" in second: + second["supersedes_revision_digest"] = getattr(first, case.digest_field) + if "payload" in second: + second["payload"] = {"revision": "B"} + return first, case.model.model_validate(second) + + +def observe(case: StoreCase, store: Any, first: DomainModel) -> dict[str, list[str]]: + args = [None if name is None else getattr(first, name) for name in case.read_fields] + current = getattr(store, case.read)(*args) + if case.read_last: + current = current[-1] if current else None + found = {case.read: [] if current is None else [str(getattr(current, case.revision_field))]} + if case.latest_list: + values = getattr(store, case.latest_list)( + *[getattr(first, key) for key in case.list_fields] + ) + found[case.latest_list] = [str(getattr(value, case.revision_field)) for value in values] + if case.read_fields[-1] is None: + historic = getattr(store, case.read)(*args[:-1], getattr(first, case.digest_field)) + assert historic is not None and historic.model_dump() == first.model_dump() + return found + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.table) +@pytest.mark.parametrize("offset", [-60, 0], ids=["rollback", "tie"]) +def test_latest_is_last_appended_revision(tmp_path: Path, case: StoreCase, offset: int) -> None: + path = tmp_path / "store.sqlite3" + engine = create_engine(f"sqlite+pysqlite:///{path.as_posix()}") + table = metadata.tables[case.table] + table.create(engine) + try: + first, second = revisions(case, offset) + store = case.store(engine) + getattr(store, case.append)(first) + getattr(store, case.append)(second) + with engine.connect() as connection: + persisted = [ + dict(row) + for row in connection.execute( + select(literal_column("rowid"), table).order_by(literal_column("rowid")) + ).mappings() + ] + assert len(persisted) == 2 + assert [datetime.fromisoformat(str(row["created_at"])) for row in persisted] == [ + STAMP, + STAMP + timedelta(seconds=offset), + ] + observed = observe(case, store, first) + finally: + engine.dispose() + reopened = create_engine(f"sqlite+pysqlite:///{path.as_posix()}") + try: + after_reopen = observe(case, case.store(reopened), first) + assert after_reopen == observed + finally: + reopened.dispose() + expected = {name: [str(getattr(second, case.revision_field))] for name in observed} + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/f03b-20261023" + / os.environ.get("THOTH_F03B_PHASE", "direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{case.table}-{'rollback' if offset else 'tie'}.json").write_text( + json.dumps( + { + "table": case.table, + "offset_seconds": offset, + "persisted": persisted, + "observed": observed, + "expected": expected, + "reopen_equal": True, + "paths": { + name: "PASS" if observed[name] == expected[name] else "FAIL" + for name in observed + }, + }, + indent=2, + default=str, + ) + + "\n", + encoding="utf-8", + ) + assert observed == expected + + +@pytest.mark.parametrize("table_name", ["plan_executions", "outcome_series"]) +@pytest.mark.parametrize( + "mode", ["higher_revision", "counter_tie_rollback", "counter_tie_same_time"] +) +def test_counter_order_precedes_time_then_uses_append_order( + tmp_path: Path, table_name: str, mode: str +) -> None: + case = next(case for case in CASES if case.table == table_name) + offset = 60 if mode == "higher_revision" else -60 if mode == "counter_tie_rollback" else 0 + first, second = revisions(case, offset) + first = case.model.model_validate({**first.model_dump(), "revision": 2}) + second = case.model.model_validate( + {**second.model_dump(), "revision": 1 if mode == "higher_revision" else 2} + ) + engine = create_engine(f"sqlite+pysqlite:///{(tmp_path / 'counter.sqlite3').as_posix()}") + metadata.tables[case.table].create(engine) + try: + store = case.store(engine) + getattr(store, case.append)(first) + getattr(store, case.append)(second) + selected = first if mode == "higher_revision" else second + expected = str(getattr(selected, case.revision_field)) + assert all(found == [expected] for found in observe(case, store, first).values()) + finally: + engine.dispose() + + +class NoModels: + calls = 0 + + def resolve(self, *, provider: str, model: str | None) -> ModelPort: + self.calls += 1 + raise AssertionError("F03b metadata flow must not resolve a model") + + +async def test_pause_uses_latest_append_while_checkpoint_history_keeps_display_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + clock = {"now": STAMP} + + def tick(self: SystemClock) -> datetime: + now = clock["now"] + clock["now"] += timedelta(microseconds=1) + return now + + monkeypatch.setattr(SystemClock, "now", tick) + models = NoModels() + runtime = create_runtime(tmp_path / "workspace", model_resolver=models) + project, thread = "project:checkpoint-clock", "thread:checkpoint-clock" + body: dict[str, object] = {"project_id": project, "thread_id": thread} + + async def call(method: str, key: str, payload: dict[str, object]) -> dict[str, Any]: + return value(await runtime.bus.dispatch(request(method, key, payload))) + + try: + await call( + "project/create", + "project", + {"project_id": project, "name": "Clock ordering", "cutoff_at": "2026-10-01T00:00:00Z"}, + ) + await call( + "thread/start", "thread", {**body, "problem": "Check clock ordering", "scope": {}} + ) + first = (await call("thread/pause", "pause-A", body))["checkpoint"] + clock["now"] = STAMP - timedelta(minutes=1) + await call("thread/resume", "resume-A", body) + second = (await call("thread/pause", "pause-B", body))["checkpoint"] + assert first["created_at"] > second["created_at"] + history = (await call("thread/checkpoint/list", "history", body))["checkpoints"] + records = SqliteThreadRuntimeStore(runtime.ledger.engine).list_checkpoints(project, thread) + expected = [ + record.model_dump(mode="json") + for record in sorted( + records, key=lambda record: (record.created_at.isoformat(), record.checkpoint_id) + ) + ] + assert history == expected + assert (await call("thread/pause", "already-paused", body))["checkpoint"] == second + resumed = await call( + "thread/resume", + "resume-B", + {**body, "expected_checkpoint_digest": second["checkpoint_digest"]}, + ) + assert resumed["execution_state"] == "IDLE" + finally: + runtime.close() + assert models.calls == 0 + + +@pytest.mark.parametrize("mode", ["higher_version", "tie_rollback", "tie_same_time", "tie_forward"]) +def test_control_current_version_ties_and_display_history(tmp_path: Path, mode: str) -> None: + case = next(case for case in CASES if case.table == "control_records") + offset = -60 if mode == "tie_rollback" else 0 if mode == "tie_same_time" else 60 + first, second = revisions(case, offset) + first = ControlRecord.model_validate( + {**first.model_dump(), "version": 3 if mode == "higher_version" else 2} + ) + second = ControlRecord.model_validate({**second.model_dump(), "version": 2}) + engine = create_engine(f"sqlite+pysqlite:///{(tmp_path / 'control.sqlite3').as_posix()}") + metadata.tables[case.table].create(engine) + try: + store = SqliteControlRecordStore(engine) + store.append(first) + store.append(second) + current = store.read(first.project_id, first.namespace, first.record_id) + latest = store.list(first.project_id, first.namespace) + history = store.list(first.project_id, first.namespace, latest_only=False) + expected = first if mode == "higher_version" else second + expected_history = tuple( + sorted( + (first, second), key=lambda record: (record.created_at.isoformat(), record.version) + ) + ) + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/f03b-20261023" + / os.environ.get("THOTH_F03B_PHASE", "direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"control-{mode}.json").write_text( + json.dumps( + { + "mode": mode, + "current": None if current is None else current.control_revision_id, + "latest": [record.control_revision_id for record in latest], + "history": [record.control_revision_id for record in history], + "expected_current": expected.control_revision_id, + "expected_history": [record.control_revision_id for record in expected_history], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + assert history == expected_history + assert current == expected + assert latest == (expected,) + finally: + engine.dispose() diff --git a/tests/unit/test_journal_completion.py b/tests/unit/test_journal_completion.py new file mode 100644 index 0000000..d145f5f --- /dev/null +++ b/tests/unit/test_journal_completion.py @@ -0,0 +1,319 @@ +"""Controlled journal state validates repair, rollback and fail-closed deferral.""" + +from collections.abc import Generator +from contextlib import contextmanager, nullcontext +from datetime import UTC, datetime, timedelta +from typing import cast + +import pytest +from pydantic import JsonValue + +from thoth.application.services.operation_journal import OperationJournal +from thoth.domain.enums import OperationState +from thoth.domain.event import Checkpoint, EventRecord +from thoth.domain.operation import OperationRecord +from thoth.domain.resource_scope import ResourceScopeError +from thoth.ports.journal import JournalCompletionUnavailable +from thoth.ports.operation import OperationStorePort +from thoth.ports.resource_scope import ResourceAccessPort +from thoth.ports.runtime import AtomicUnitOfWorkPort +from thoth.protocol.journal_completion import JournalCompletion, operation_result_digest +from thoth.protocol.journal_completion_methods import SUPPLEMENTABLE_METHODS, SUPPORTED_RESULT_FORMS + +BASE = datetime(2026, 10, 19, tzinfo=UTC) +METHOD = "project/metadata/update" + + +def test_measured_methods_and_saved_forms_match_reviewed_fixture() -> None: + import json + from pathlib import Path + + from thoth.protocol.read_queries import READ_QUERY_METHODS + from thoth.protocol.registry import PUBLIC_METHODS + + fixture = json.loads( + (Path(__file__).parents[1] / "fixtures/supplementable_methods.json").read_text( + encoding="utf-8" + ) + ) + assert len(SUPPLEMENTABLE_METHODS) == 20 + assert sorted(SUPPLEMENTABLE_METHODS) == fixture["methods"] + assert set(PUBLIC_METHODS) >= SUPPLEMENTABLE_METHODS + assert not SUPPLEMENTABLE_METHODS.intersection(READ_QUERY_METHODS) + assert not SUPPLEMENTABLE_METHODS.intersection( + { + "thread/start", + "thread/input", + "thread/steer", + "thread/pause", + "thread/resume", + "thread/stop", + "revision/restore", + "revision/restore/apply", + "revision/changeSet/commit", + } + ) + assert { + method: sorted(sorted(keys) for keys in forms) + for method, forms in SUPPORTED_RESULT_FORMS.items() + } == fixture["result_key_shapes"] + assert fixture["evidence"]["path"] == "outputs/refactor-vj-20261019/method-evidence.json" + + +class Clock: + def __init__(self) -> None: + self.value = BASE - timedelta(seconds=1) + self.fixed = False + self.calls = 0 + + def now(self) -> datetime: + value = self.value + self.calls += 1 + if not self.fixed: + self.value += timedelta(microseconds=1) + return value + + +class Ids: + def __init__(self) -> None: + self.calls = 0 + self.tag = "z" + + def new(self, prefix: str) -> str: + self.calls += 1 + return f"{prefix}:{self.tag}-{self.calls:05}" + + +class State: + def __init__(self) -> None: + result: dict[str, JsonValue] = dict.fromkeys(SUPPORTED_RESULT_FORMS[METHOD][0]) + result.update(schema_version="1.0.0", project_id="project:j-unit", name="Saved", revision=1) + self.operation = OperationRecord( + operation_id="operation:j-unit", + project_id="project:j-unit", + method=METHOD, + idempotency_key="key", + scope_digest="a" * 64, + state=OperationState.SUCCEEDED, + result=result, + resource_uses=(), + created_at=BASE - timedelta(seconds=1), + completed_at=BASE, + ) + self.events: list[EventRecord] = [] + self.points: list[Checkpoint] = [] + self.fail_write = False + self.deny = False + self.in_transaction = False + + def read(self, operation_id: str) -> OperationRecord: + assert operation_id == self.operation.operation_id + return self.operation + + def require_operation(self, operation: OperationRecord) -> None: + assert self.in_transaction and operation.operation_id == self.operation.operation_id + if self.deny: + raise ResourceScopeError("RESOURCE_SCOPE_READ_DENIED") + + def list_events(self, operation_id: str) -> tuple[EventRecord, ...]: + return tuple(sorted(self.events, key=lambda e: (e.created_at, e.event_id))) + + def list_checkpoints(self, operation_id: str) -> tuple[Checkpoint, ...]: + return tuple(self.points) + + def latest_checkpoint(self, operation_id: str) -> Checkpoint | None: + return self.points[-1] if self.points else None + + def insert_event(self, event: EventRecord) -> None: + self.events.append(event) + if self.fail_write: + raise JournalCompletionUnavailable("synthetic write failure") + + def insert_checkpoint(self, checkpoint: Checkpoint) -> None: + self.points.append(checkpoint) + + @contextmanager + def transaction(self) -> Generator[None]: + before = self.events[:], self.points[:] + self.in_transaction = True + try: + yield + except BaseException: + self.events[:], self.points[:] = before + raise + finally: + self.in_transaction = False + + @contextmanager + def completion_transaction(self, uow: AtomicUnitOfWorkPort) -> Generator[None]: + with uow.transaction(): + yield + + +def setup() -> tuple[State, Clock, Ids, OperationJournal, JournalCompletion]: + state, clock, ids = State(), Clock(), Ids() + journal = OperationJournal(state, clock, ids) + journal.append( + project_id=state.operation.project_id, + operation_id=state.operation.operation_id, + event_type="operation.started", + payload={"method": METHOD}, + ) + clock.value = BASE + timedelta(seconds=10) + completion = JournalCompletion( + workspace_key="workspace:unit", + pin_workspace=lambda: nullcontext(True), + operations=cast(OperationStorePort, state), + journal=journal, + store=state, + uow=state, + resource_access=cast(ResourceAccessPort, state), + ) + return state, clock, ids, journal, completion + + +def test_repair_is_marked_and_second_call_allocates_and_writes_nothing() -> None: + state, clock, ids, _, completion = setup() + original = state.operation.model_dump_json() + outcome = completion.attempt(state.operation) + assert outcome.status == "SUPPLEMENTED" and outcome.added == 3 + assert state.operation.model_dump_json() == original + assert [e.event_type for e in state.events] == [ + "operation.started", + "project/metadata/updated", + "operation.succeeded", + ] + for item in [*state.events[1:], *state.points]: + assert item.payload["supplemented"] is True + assert item.payload["supplemented_for"] == BASE.isoformat() + assert item.created_at >= BASE + timedelta(seconds=10) + before = state.events[:], state.points[:], clock.calls, ids.calls + assert completion.attempt(state.operation).status == "COMPLETE" + assert (state.events, state.points, clock.calls, ids.calls) == before + + +@pytest.mark.parametrize("marked", [False, True]) +def test_partial_original_or_supplemented_tail_keeps_existing_bytes(marked: bool) -> None: + state, _, _, journal, completion = setup() + assert state.operation.result is not None + digest = operation_result_digest(state.operation.result) + mark: dict[str, JsonValue] = ( + {"supplemented": True, "supplemented_for": BASE.isoformat()} if marked else {} + ) + journal.checkpoint( + operation_id=state.operation.operation_id, + payload={"state": "SUCCEEDED", "result_digest": digest, **mark}, + ) + journal.append( + project_id=state.operation.project_id, + operation_id=state.operation.operation_id, + event_type="project/metadata/updated", + payload={"method": METHOD, "result_digest": digest, **mark}, + ) + before = ( + [e.model_dump_json() for e in state.events], + [p.model_dump_json() for p in state.points], + ) + outcome = completion.attempt(state.operation) + assert outcome.status == "SUPPLEMENTED" and outcome.added == 1 + assert [e.model_dump_json() for e in state.events[:2]] == before[0] + assert [p.model_dump_json() for p in state.points] == before[1] + + +@pytest.mark.parametrize( + "case", + [ + "unknown-method", + "failed", + "missing-result", + "missing-completed-at", + "unknown-shape", + "unknown-schema", + ], +) +def test_unsupported_saved_records_are_not_repaired(case: str) -> None: + state, clock, ids, _, completion = setup() + changes: dict[str, object] = {} + if case == "unknown-method": + changes["method"] = "revision/changeSet/commit" + if case == "failed": + changes.update( + state=OperationState.FAILED, result=None, error={"reason_code": "OPERATION_OWNER_GONE"} + ) + if case == "missing-result": + changes["result"] = None + if case == "missing-completed-at": + changes["completed_at"] = None + if case == "unknown-shape": + changes["result"] = {"unknown": True} + if case == "unknown-schema": + changes["schema_version"] = "2.0.0" + state.operation = state.operation.model_copy(update=changes) + before = clock.calls, ids.calls, state.events[:], state.points[:] + assert completion.attempt(state.operation).status in {"DEFERRED", "NOT_ELIGIBLE"} + assert (clock.calls, ids.calls, state.events, state.points) == before + + +@pytest.mark.parametrize( + "case", ["chain", "duplicate", "payload", "publisher", "checkpoints", "checkpoint-payload"] +) +def test_ambiguous_or_broken_existing_journal_defers_without_writes(case: str) -> None: + state, clock, ids, journal, completion = setup() + assert state.operation.result is not None + digest = operation_result_digest(state.operation.result) + if case == "chain": + state.events[0] = state.events[0].model_copy(update={"event_digest": "f" * 64}) + if case in {"duplicate", "payload", "publisher"}: + for _ in range(2 if case == "duplicate" else 1): + journal.append( + project_id=state.operation.project_id, + operation_id=state.operation.operation_id, + event_type="research.terminal" + if case == "publisher" + else "project/metadata/updated", + payload={ + "method": METHOD, + "result_digest": "f" * 64 if case == "payload" else digest, + }, + ) + if case in {"checkpoints", "checkpoint-payload"}: + for _ in range(2 if case == "checkpoints" else 1): + journal.checkpoint( + operation_id=state.operation.operation_id, + payload={ + "state": "SUCCEEDED", + "result_digest": "f" * 64 if case == "checkpoint-payload" else digest, + }, + ) + before = state.events[:], state.points[:], clock.calls, ids.calls + assert completion.attempt(state.operation).status == "DEFERRED" + assert (state.events, state.points, clock.calls, ids.calls) == before + + +@pytest.mark.parametrize("case", ["rollback-clock", "equal-time-reversed-id", "storage"]) +def test_new_batch_is_rolled_back_on_bad_order_or_storage_failure(case: str) -> None: + state, clock, ids, _, completion = setup() + if case == "rollback-clock": + clock.value = BASE - timedelta(seconds=5) + if case == "equal-time-reversed-id": + clock.value, clock.fixed, ids.tag = state.events[0].created_at, True, "a" + if case == "storage": + state.fail_write = True + before = state.events[:], state.points[:], state.operation.model_dump_json() + outcome = completion.attempt(state.operation) + assert outcome.status == "DEFERRED" + assert (state.events, state.points, state.operation.model_dump_json()) == before + + +@pytest.mark.parametrize("case", ["denied", "canonical-change"]) +def test_authority_and_canonical_failures_are_not_swallowed(case: str) -> None: + state, _, _, _, completion = setup() + original = state.operation + if case == "denied": + state.deny = True + else: + state.operation = original.model_copy(update={"scope_digest": "c" * 64}) + before = state.events[:], state.points[:] + with pytest.raises(ResourceScopeError): + completion.attempt(original) + assert (state.events, state.points) == before diff --git a/tests/unit/test_journal_completion_gate.py b/tests/unit/test_journal_completion_gate.py new file mode 100644 index 0000000..2dc4a04 --- /dev/null +++ b/tests/unit/test_journal_completion_gate.py @@ -0,0 +1,92 @@ +"""The journal gate is reentrant, shared and leak-free; result digest bytes are frozen.""" + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Event + +import pytest +from pydantic import JsonValue + +from thoth.domain.errors import CanonicalizationError +from thoth.protocol import journal_completion_gate as gates +from thoth.protocol.journal_completion import operation_result_digest + +VECTORS = json.loads( + (Path(__file__).parents[1] / "fixtures/operation_result_digest_vectors.json").read_text( + encoding="utf-8" + ) +)["vectors"] + + +@pytest.mark.parametrize("vector", VECTORS) +def test_digest_bytes_match_cp56_vectors(vector: dict[str, object]) -> None: + from typing import cast + + actual = operation_result_digest(cast(dict[str, JsonValue], vector["value"])) + assert actual.encode("ascii") == str(vector["digest"]).encode("ascii") + + +def test_binary_float_still_fails_canonicalization() -> None: + with pytest.raises(CanonicalizationError, match="binary float"): + operation_result_digest({"value": 0.25}) + + +def test_reentrant_gate_releases_after_exception() -> None: + key = ("workspace:gate-unit", "operation:gate-unit") + entries = vars(gates)["_entries"] + assert key not in entries + with pytest.raises(RuntimeError, match="injected"), gates.journal_gate(*key): + entry = entries[key] + assert entry.users == 1 + with gates.journal_gate(*key): + assert entries[key] is entry and entry.users == 2 + assert entry.users == 1 + raise RuntimeError("injected") + assert key not in entries + + +def test_two_threads_share_gate_and_remove_last_reference() -> None: + key = ("workspace:threads", "operation:threads") + first_entered, second_attempted, release_first, second_entered = (Event() for _ in range(4)) + order: list[str] = [] + + def first() -> None: + with gates.journal_gate(*key): + first_entered.set() + assert release_first.wait(timeout=5) + order.append("first") + + def second() -> None: + second_attempted.set() + with gates.journal_gate(*key): + second_entered.set() + order.append("second") + + with ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(first) + assert first_entered.wait(timeout=5) + b = pool.submit(second) + try: + assert second_attempted.wait(timeout=5) + assert not second_entered.wait(timeout=0.05) + finally: + release_first.set() + a.result(timeout=5) + b.result(timeout=5) + assert order == ["first", "second"] + assert key not in vars(gates)["_entries"] + + +def test_same_operation_in_other_workspace_uses_a_different_gate() -> None: + with ( + gates.journal_gate("workspace:a", "operation:same"), + ThreadPoolExecutor(max_workers=1) as pool, + ): + + def other() -> str: + with gates.journal_gate("workspace:b", "operation:same"): + return "entered" + + assert pool.submit(other).result(timeout=5) == "entered" + assert not {key for key in vars(gates)["_entries"] if key[1] == "operation:same"}