diff --git a/SOURCE_MANIFEST.json b/SOURCE_MANIFEST.json index 5c8f14a..32adc81 100644 --- a/SOURCE_MANIFEST.json +++ b/SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schema_version": "thoth-source-preview-manifest-v1", - "created_at": "2026-10-19T13:28:27.492646+00:00", + "created_at": "2026-10-22T18:37:46.948775+00:00", "license": "MIT", "git_history_included": false, "full_suite_verified": false, @@ -411,6 +411,11 @@ "sha256": "9edd2a0cccfbd903f70a85787f05274b402dee7c6e6b4e454078ab8dfcfe97d3", "bytes": 1709 }, + { + "path": "apps/web/src/api/rpcClient.test.ts", + "sha256": "aad467a95444ab712ca5cede7034d6acb4ac0d4c94f58c77355a7896123d1ebf", + "bytes": 2163 + }, { "path": "apps/web/src/api/rpcClient.ts", "sha256": "92a77302b04086eca3e3c422248b81017586cbaaca4054370149043b16407c69", @@ -1878,7 +1883,7 @@ }, { "path": "config/module-responsibility-budget.json", - "sha256": "2c8eee608d261d612e04010f1e3e18bd1df3549a91b91d323beee60316924f58", + "sha256": "6992a0c3aa7cc3822aa5c7eaa461e04961ba89809ae22797c914676469e5655a", "bytes": 8580 }, { @@ -1948,13 +1953,13 @@ }, { "path": "docs/KNOWN_LIMITATIONS.ko.md", - "sha256": "777febcf50b13b2878658fe826e34c0230d3d9930efbf9cea72897facfcefe60", - "bytes": 8864 + "sha256": "ce5cc699dc080560681b09f0f7790365e13d57922c0f9d4ba117152b6d297e27", + "bytes": 10028 }, { "path": "docs/KNOWN_LIMITATIONS.md", - "sha256": "f294c9aaaefcca87c0d61f9704d0bbf3a9a97568bcf00570467eec74128ce113", - "bytes": 7626 + "sha256": "6d7bbcb9c5b7e392c19b8d44d2dc7bccb03741cae9d524983f856a6efeb690d4", + "bytes": 8541 }, { "path": "docs/KNOWN_TEST_FAILURES.json", @@ -3923,8 +3928,8 @@ }, { "path": "src/thoth/adapters/storage/action.py", - "sha256": "26f9db308116e146d36c5cb173c6e49bdce714e37f77a98ecfe2c41aa35ad1f9", - "bytes": 16247 + "sha256": "bc81daab38f70a8444d20b1172b701b1b2bf8614759389235710e67c6d5cafec", + "bytes": 15954 }, { "path": "src/thoth/adapters/storage/action_schema.py", @@ -3993,8 +3998,8 @@ }, { "path": "src/thoth/adapters/storage/control_record.py", - "sha256": "ea1b264b5a0fe73086e756f36482cff17e2f5753de32af51a69f819cfc11e157", - "bytes": 4345 + "sha256": "1683f971c489ea923de3f46a5799336e05a96136291756175d1e595ddc5797da", + "bytes": 4586 }, { "path": "src/thoth/adapters/storage/conversation.py", @@ -4003,8 +4008,8 @@ }, { "path": "src/thoth/adapters/storage/criterion_contract.py", - "sha256": "523bc48f72511909c0ca29e65bcecc217a9b00b911444fe45f04ef9e42f2c2ce", - "bytes": 17105 + "sha256": "000584b6882d1d688f3072fade00407a10d6735fc8cb10584c1d3b389ecb1fed", + "bytes": 17000 }, { "path": "src/thoth/adapters/storage/criterion_schema.py", @@ -4013,8 +4018,8 @@ }, { "path": "src/thoth/adapters/storage/decision_object.py", - "sha256": "e1509f97eaab1b95bf928c4e6b41faa7917e6a01dc792dcf7b7aa5810d198b7d", - "bytes": 16633 + "sha256": "58ccbf0e263e6b41b13fc57bb6a742e964a74193190a83bd1096f1e6e67bf77d", + "bytes": 16718 }, { "path": "src/thoth/adapters/storage/decision_object_schema.py", @@ -4038,13 +4043,13 @@ }, { "path": "src/thoth/adapters/storage/events.py", - "sha256": "0ed076d668ab5abe9d912894e01401941e92ca44059ab518efd9159d0376b3d1", - "bytes": 4340 + "sha256": "80ec394619aae2688f935810058b445aa301d5d7134535d6352cc8b7ab606af2", + "bytes": 5757 }, { "path": "src/thoth/adapters/storage/evidence_graph.py", - "sha256": "292302b0e7572426659d5dc5ea73025e40b310c0079e13ed7e03fd0bce1c40cc", - "bytes": 19720 + "sha256": "bc721ea2d9b4a4b2202dd036adb0d3cf2ce50609b63d2a974d6eb7dad66944c5", + "bytes": 19611 }, { "path": "src/thoth/adapters/storage/evidence_schema.py", @@ -4053,8 +4058,8 @@ }, { "path": "src/thoth/adapters/storage/execution.py", - "sha256": "e1ce36bc08f88942880ec9c25f852aea1bdcfd30ab616696c90299dc2e90eb85", - "bytes": 10260 + "sha256": "d9a78985468c0e40f4821d284249928651f94f49cfeb03efc33ce897c679aa41", + "bytes": 10381 }, { "path": "src/thoth/adapters/storage/execution_schema.py", @@ -4093,8 +4098,8 @@ }, { "path": "src/thoth/adapters/storage/hypothesis.py", - "sha256": "3be522d8cd6ed0705c7055f9e333e8d59c9ec605c6cdbc7c07cde3996d060a68", - "bytes": 19201 + "sha256": "01f25bafe4267fbd190739db9abff21aeeeff2ad2955c5a99fa7e80bfed6fcf7", + "bytes": 18890 }, { "path": "src/thoth/adapters/storage/hypothesis_schema.py", @@ -4138,8 +4143,8 @@ }, { "path": "src/thoth/adapters/storage/outcome.py", - "sha256": "cdd121cfb79d8d4dd7559d889b18e5962a07e12375e895cd4a5111258a4b5b9d", - "bytes": 15233 + "sha256": "bf730a9c6e46203dc07aa21e34b0e419564356ee22c5a101b036b48860e2dcd0", + "bytes": 15643 }, { "path": "src/thoth/adapters/storage/outcome_schema.py", @@ -4223,8 +4228,8 @@ }, { "path": "src/thoth/adapters/storage/thread_runtime.py", - "sha256": "c1e2f9c13bdcc90af08369e68f39f5e9b945036aaf42ff6637ed4c4a3c933f74", - "bytes": 7879 + "sha256": "0153010a13fb79114d58c8f2fe30043b2eea4a7b58e9ba116061b5bccaf35de6", + "bytes": 7871 }, { "path": "src/thoth/adapters/storage/threads.py", @@ -4238,8 +4243,8 @@ }, { "path": "src/thoth/adapters/storage/workspace_owner.py", - "sha256": "ec0f5d99b5e71ba00fc16f988ace72779788cdaa0a9f1ac011646ebf5f213584", - "bytes": 6935 + "sha256": "7138a59c44a2c25e9d5a79df5112caff82ebb286ab61a6c7c1c36f250ea78989", + "bytes": 7097 }, { "path": "src/thoth/adapters/storage/workspace_setup.py", @@ -4468,8 +4473,8 @@ }, { "path": "src/thoth/application/commands/memory_full.py", - "sha256": "5ea4c7510c781623f59a5e8ea64688470c33543913d448613511f9091117aa97", - "bytes": 22235 + "sha256": "ac7bf6c342d1d293684a0026e69e68ff8c18e9ff30b3e56cda45a17e17a30881", + "bytes": 22173 }, { "path": "src/thoth/application/commands/memory_inputs.py", @@ -4481,6 +4486,11 @@ "sha256": "d893dd4b581e95e6eaa4757fbec7ac9a3333238b9080d137a6fa22c056786d15", "bytes": 1815 }, + { + "path": "src/thoth/application/commands/memory_records.py", + "sha256": "53bea1a5b4742c4a76e576e2a351dcb1082aa7d25b718fd54bdf841cbbec0633", + "bytes": 695 + }, { "path": "src/thoth/application/commands/memory_revisions.py", "sha256": "0a79894a19107114b6669f342b178a0fe62157a7a02434307a41a4fd27ef88c9", @@ -4563,8 +4573,8 @@ }, { "path": "src/thoth/application/commands/reference.py", - "sha256": "469d3b8fe4f47a28fe040fad4619f1b27650cdf14acfbdda07a18caa9f6ac37b", - "bytes": 7351 + "sha256": "0dc5aca4c53c8c75879c95fd6f39fc5d247f03f669dbae98d56d7db8a8b260b1", + "bytes": 7740 }, { "path": "src/thoth/application/commands/research_followup.py", @@ -4668,8 +4678,8 @@ }, { "path": "src/thoth/application/commands/threads.py", - "sha256": "4335b93071fcba7d318304e7e1dbd239c0a573f1cbe8343e57af0561fa06b6a1", - "bytes": 23304 + "sha256": "329fcd5eda0f860baeb4f341f64e09797c0ece261ccb5fbdc4e62c08a7495661", + "bytes": 23439 }, { "path": "src/thoth/application/commands/trace_closure.py", @@ -5423,8 +5433,8 @@ }, { "path": "src/thoth/application/services/r2_closed_loop.py", - "sha256": "0970aa27c14e0eb4f905f883120500f00c599d73f9cbe0944ca2f97726c1f74f", - "bytes": 24940 + "sha256": "4ef8fbb555f3e2d1864902f668452b9e410161f9324c63ba82df8f314a97aa36", + "bytes": 25475 }, { "path": "src/thoth/application/services/r2_context_preflight.py", @@ -5436,6 +5446,11 @@ "sha256": "2cb3a97cc78385c42e8d8b88ce87497c5df979f6758a0d8577b1d038fec743b7", "bytes": 7373 }, + { + "path": "src/thoth/application/services/r2_failure_publication.py", + "sha256": "625fe6a36fbb42eaa599a8622786239aabb7939c6d5adab33d7a030c914122d4", + "bytes": 2072 + }, { "path": "src/thoth/application/services/r2_outcome_projection.py", "sha256": "fec137ddbb3297dce6ab21e24ccfcd7d18a60dead1710638881d54b6d292bd59", @@ -5443,8 +5458,8 @@ }, { "path": "src/thoth/application/services/r2_recovery_records.py", - "sha256": "912fcbbcf07a91d22620039eb49f8ecf7da7b40330924f36432a09027c978bfb", - "bytes": 8894 + "sha256": "f73b0035ba36fc82b91f645eba2b57980665339bd4ecf3cdf9a8d2f62bc74b23", + "bytes": 10222 }, { "path": "src/thoth/application/services/r2_sandbox_compiler.py", @@ -5471,6 +5486,11 @@ "sha256": "aec4728dab5a7ae2a5ca20bcf0fc8264f31fa2629bfc63ebf9b40b59d1bdb188", "bytes": 7823 }, + { + "path": "src/thoth/application/services/reference_failure.py", + "sha256": "1bd6711e072fc06027a80d014b5dc4454556ca9e60bdaad0907a504f312e425f", + "bytes": 4598 + }, { "path": "src/thoth/application/services/reference_grounding.py", "sha256": "c39afc2d0e5d37ff204870a5886ad54729a95ac36cad60b9641ea6e1598cf525", @@ -5523,8 +5543,8 @@ }, { "path": "src/thoth/application/services/research_attempt_runner.py", - "sha256": "868fab123ff6482f97281f1d853e8208324161a7f21227da76a3b4ff1e333acf", - "bytes": 18844 + "sha256": "5e6499afabe6ea7d2d10f80123e8070f55eee4a265c02d2c06901e0d329dc088", + "bytes": 18958 }, { "path": "src/thoth/application/services/research_basis_capture.py", @@ -5743,8 +5763,8 @@ }, { "path": "src/thoth/application/services/research_termination.py", - "sha256": "25c78dca2407f3b5a7514f5bd22291f7f5b5ee304b408a4f33bcb232b576f470", - "bytes": 5501 + "sha256": "8c4e550c2d90d486a1f3c548ceb300ae72364cb89b82d8a9d2703cc222f2d224", + "bytes": 5824 }, { "path": "src/thoth/application/services/research_thread_read_snapshot.py", @@ -5858,8 +5878,8 @@ }, { "path": "src/thoth/application/services/sandbox_service.py", - "sha256": "669690aa50057b4bb208b360381f12f84bf7356f25ca59cb50382b75bd11d3bf", - "bytes": 25937 + "sha256": "0b053c9acbcdbdd2344f6c29035930ad63d0919f861008aad56b40eca95ce25c", + "bytes": 26030 }, { "path": "src/thoth/application/services/scoped_artifacts.py", @@ -6168,8 +6188,8 @@ }, { "path": "src/thoth/apps/reference_composition.py", - "sha256": "53e72e2f0a9ab0753fd79720fc94f28e1fb364bfb2e71f1bf29d4356e811ab5e", - "bytes": 1652 + "sha256": "5b4f185fb11a0f6140349c517163cf5876666843b73b1635c90c2551acdeebef", + "bytes": 1668 }, { "path": "src/thoth/apps/research_entry_composition.py", @@ -6188,8 +6208,8 @@ }, { "path": "src/thoth/apps/resource_scope_composition.py", - "sha256": "f29ad10251d86251ac70f5c6330ef08db1fefa631b3ad66c7e8ed41d0bdfca5a", - "bytes": 7204 + "sha256": "478622520453bed77125dcd5da84133618a442550ac87cdc9fc80bc9b3a7f92a", + "bytes": 7372 }, { "path": "src/thoth/apps/restore_profiles.py", @@ -6208,8 +6228,8 @@ }, { "path": "src/thoth/apps/runtime_entry.py", - "sha256": "b9bc5d885152e0fe4bc084678b8f17810facb37313a4c46651f1ce481413b50d", - "bytes": 6005 + "sha256": "6a81df4e30b133962fa1051f6effb2ceef2e68a33fe285c418189c393e036f6d", + "bytes": 6763 }, { "path": "src/thoth/apps/runtime_registration.py", @@ -7048,8 +7068,8 @@ }, { "path": "src/thoth/ports/journal.py", - "sha256": "6eabc143facec100dc053fe98f57b024e31697ae9962afee7e1e38b0ba32f692", - "bytes": 694 + "sha256": "87985bde73c736d1be3f3d06ba30fe33272fe28f9ff12b5019d7b0381a779fbb", + "bytes": 1368 }, { "path": "src/thoth/ports/ledger.py", @@ -7223,14 +7243,29 @@ }, { "path": "src/thoth/protocol/bus.py", - "sha256": "5a6f0012d36f4e719080566385916ce7441977baaaaaa6127f970b8b2f2d06a7", - "bytes": 31488 + "sha256": "95e281e6ddbebbe87abde04fc37b437785d8511cc9138bf2551bd156c674624b", + "bytes": 31656 }, { "path": "src/thoth/protocol/deferred.py", "sha256": "c4ec6f0d306c20b48ded1a7ebdee4d69f1d0e91d294792f39d0065472b40fdc1", "bytes": 927 }, + { + "path": "src/thoth/protocol/journal_completion.py", + "sha256": "f0d2a939b376d8f0998feaff4d52102fbafe0a9fe51da622e19242402315fbea", + "bytes": 9840 + }, + { + "path": "src/thoth/protocol/journal_completion_gate.py", + "sha256": "e6bbd755fcb053e7d25f026f21e115e974066fc2bd6ebc5daaebeb3b2c333468", + "bytes": 917 + }, + { + "path": "src/thoth/protocol/journal_completion_methods.py", + "sha256": "4a19219b6baf0c179a0ae14ddd26502ff20399b31f219ca9dd10a6e5298d64b1", + "bytes": 4365 + }, { "path": "src/thoth/protocol/jsonrpc.py", "sha256": "8da5f77e17afb3e73e3355de50dfa3a857366472a7080d380d16daec88d2b67c", @@ -7261,6 +7296,11 @@ "sha256": "8f6da0b375e71715fe8ab4b959b8c41b9be1f7b826075ff0bc937dad4f0b8c73", "bytes": 8187 }, + { + "path": "src/thoth/protocol/operation_scope.py", + "sha256": "87f430719f2bdef2e103edaa3c348937b71c9cb9cc36f96e2495724a4abe175d", + "bytes": 470 + }, { "path": "src/thoth/protocol/read_queries.py", "sha256": "17a7460e4c4be9e9b07c2cda5240346e758fa6bc1410e259c02596ed6ea81333", @@ -7881,6 +7921,11 @@ "sha256": "7f62d3b3ffc64c620be19ca6b0786d9a775c1c3be198727c281e7cfad0e67366", "bytes": 1242 }, + { + "path": "tests/fixtures/operation_result_digest_vectors.json", + "sha256": "812d4554d29fb41871c1ba9b49a577881112001ce0eedc51bb7bbe4851869838", + "bytes": 1022 + }, { "path": "tests/fixtures/outcome_response_golden.json", "sha256": "8994fb66738ff2926e0a17586b73f1156fe433d5d570c4671b5e56a6b3ed55fb", @@ -7893,8 +7938,8 @@ }, { "path": "tests/fixtures/r2_closed_loop_golden.json", - "sha256": "4afc05462d53e7324146f0bc6ae38c5903792c5cf9ed06b16774df3678d4be63", - "bytes": 653428 + "sha256": "594820e171b9221fc34c36b66ce414f75df601ac64968e7af9bb56bd907caaa1", + "bytes": 653944 }, { "path": "tests/fixtures/recursive_improvement_golden.json", @@ -7903,8 +7948,8 @@ }, { "path": "tests/fixtures/reference_service_golden.json", - "sha256": "9f9e0ab91a4150a6c526ed270fcb84e633fb4fb822cde1b4cc88428d1f7909e2", - "bytes": 1126800 + "sha256": "9f29e71a60114ba4716835467842b86311d314347694036adf3070496b63af31", + "bytes": 1127566 }, { "path": "tests/fixtures/resource_scope_records_golden.json", @@ -7966,6 +8011,11 @@ "sha256": "46cbc6121770020d167dc3c24bfa4cfd0716c67e94a1b1b73745985303314a93", "bytes": 3758 }, + { + "path": "tests/fixtures/supplementable_methods.json", + "sha256": "61e0c7811e4999eb1e79dc3f639ea723c73becc5c1fe4033eacf4d71146f6cff", + "bytes": 7309 + }, { "path": "tests/fixtures/trace_csv/README.txt", "sha256": "164fbf3abc1fe39f18ef3a5ab1645da3030dcebf6ae6a137515ba219340e60e2", @@ -8051,11 +8101,21 @@ "sha256": "32357695636c28b0bef0b8d3c0d9e861dee2ab516b62192bd8c0f4ec8e40d804", "bytes": 31117 }, + { + "path": "tests/integration/journal_completion_helpers.py", + "sha256": "51bc70fd4bdd2d2a8bd357bda4611231d79bb9a9a91aa9857dd880f0de8c82bc", + "bytes": 2655 + }, { "path": "tests/integration/migration_head_helpers.py", "sha256": "558317d21741346f81c115e2d2c60ea461d0831b60319593509756f470db7998", "bytes": 698 }, + { + "path": "tests/integration/p_r2_failure_helpers.py", + "sha256": "0e2c5aa23834785508af97a8638b8619e80d10b641812ced3fcb64980f09a7d9", + "bytes": 2902 + }, { "path": "tests/integration/paired_evaluation_helpers.py", "sha256": "82966f164bd44062dc3655b5db0d7bfcb47e428c93fe9b34a11ef5ffa1ca8f4d", @@ -8731,6 +8791,26 @@ "sha256": "5e64d55d216651b3c1eb2669c9bb4eb04b881cd15e12dcb05d288af1427f98f0", "bytes": 7761 }, + { + "path": "tests/integration/test_journal_completion_concurrency.py", + "sha256": "fa144f687f36b5134ed6d31fe723af56173c9a7504eba9fa5612dec5043ae0ef", + "bytes": 5320 + }, + { + "path": "tests/integration/test_journal_completion_faults.py", + "sha256": "6e21c08dbdc31a505437a8597331344aa3e66235fdf964db9b0cceaa1fe9c1e0", + "bytes": 6324 + }, + { + "path": "tests/integration/test_journal_completion_process.py", + "sha256": "36d4d0c86b33b16b0a7821e0dcbae49f19ee33fe0c776bc81b7e071075c8c470", + "bytes": 6171 + }, + { + "path": "tests/integration/test_journal_completion_replay.py", + "sha256": "a797e1e289a35697e872c86a56be0943c41e3711ae139663273d2debcdd692be", + "bytes": 6877 + }, { "path": "tests/integration/test_judgment_records.py", "sha256": "7778ae97e6d23da3efd9d5e81658c2c63d05e2fa4fe02949fed93e3e4fc2ccbb", @@ -8828,8 +8908,8 @@ }, { "path": "tests/integration/test_m3_hypothesis_first_write_reachability.py", - "sha256": "273625a7ce30feaa6598b056d74d40a641271d3239db9edafe73c92971e63733", - "bytes": 18476 + "sha256": "edcbd23477dc1be0e1d4a66ac5d4d064d246e31ec8104e18427b19eade71264e", + "bytes": 18650 }, { "path": "tests/integration/test_m3_hypothesis_mutation_atomicity.py", @@ -8873,13 +8953,13 @@ }, { "path": "tests/integration/test_m3_substantive_appraisal_atomicity.py", - "sha256": "5810761fbcebd2723ec129e5084b2e4fa22f113bbfe69bdbad55a303bd8545f6", - "bytes": 17895 + "sha256": "684854ad58dfb159f5bfe2fca75bc4ea8ed3c9becec9a42f99768728e1e5e2e7", + "bytes": 18069 }, { "path": "tests/integration/test_m4_memory_control_multiwrite_observation.py", - "sha256": "49663656a8527b1f8c10311ef1edc731be6bee31e8af56293c4af46b649883ef", - "bytes": 18382 + "sha256": "df0eb3fb72777694a1dcbdd11750341e35915de8150b66649c00beb176376ba3", + "bytes": 17799 }, { "path": "tests/integration/test_m5_exposure_service_atomicity.py", @@ -9048,8 +9128,8 @@ }, { "path": "tests/integration/test_operation_journal_failure_boundaries.py", - "sha256": "7b269e5c1382f64823122be46004ea6e0679725d3ba6b6d9644523596e37a927", - "bytes": 13419 + "sha256": "bf2b8ff1039d1455e4702ec92115b4785ee2a54239f9fefe3d7fa51715459d7b", + "bytes": 14433 }, { "path": "tests/integration/test_operation_owner_atomicity.py", @@ -9091,6 +9171,21 @@ "sha256": "ba727b080bd12284ca1e80604154a79e8d53fdd310a0a67cc673536d73146785", "bytes": 7738 }, + { + "path": "tests/integration/test_p_memory_atomicity.py", + "sha256": "a3e4b3f143fa1c3f3a0e5b685f0ff998acb05ae5c15145f1350ca465efdc0467", + "bytes": 5637 + }, + { + "path": "tests/integration/test_p_r2_failure_atomicity.py", + "sha256": "cbf8e3a7140850802a253d72f131de415716487e047dadd6cd850a535f3de231", + "bytes": 10405 + }, + { + "path": "tests/integration/test_p_reference_failure_notice.py", + "sha256": "f8c40869a0331027c8375af1c9d48bbb6a79af66baabcb50836bcc73cafc4cfa", + "bytes": 11541 + }, { "path": "tests/integration/test_paired_evaluation_execution.py", "sha256": "d68853afb5cfe88014dd7dc3cfa446b5580a9132da5e21ab7d40e692ed87db5c", @@ -9756,6 +9851,11 @@ "sha256": "1620bfa1c34d9c17cc5ed1477893596c720b097df597b99afa70f2e4ac919ba6", "bytes": 18126 }, + { + "path": "tests/integration/test_storage_clock_currentness.py", + "sha256": "5fda6124d80d745b9fce8bd8095dc191a3b9d9a5931dd02cda94cfd52e05600f", + "bytes": 23428 + }, { "path": "tests/integration/test_storage_migration_ddl_contract.py", "sha256": "55e246ca9efa5312baa9c7b39dc22361a13efd55680486a75cf62dbdca171c21", @@ -11146,6 +11246,16 @@ "sha256": "88e55392895bb63a1da1ad7c26abbee88778d813194bd29779ecb51dea18020f", "bytes": 10661 }, + { + "path": "tests/unit/test_journal_completion.py", + "sha256": "37896a225b6fc37e3d80fa962f895ad9848840f7f0377f3cf763d6a5433fdc1f", + "bytes": 12376 + }, + { + "path": "tests/unit/test_journal_completion_gate.py", + "sha256": "471f7cfebed4b6694007c79e90f2c565875bee8f537ad650046d480b9d6e7928", + "bytes": 3179 + }, { "path": "tests/unit/test_lesson_recall_cases.py", "sha256": "a20593ab7789905988c265ca75f5297575d57e39f3d5c81ee8980bb110dd1ddb", diff --git a/apps/web/src/api/rpcClient.test.ts b/apps/web/src/api/rpcClient.test.ts new file mode 100644 index 0000000..2aee9e2 --- /dev/null +++ b/apps/web/src/api/rpcClient.test.ts @@ -0,0 +1,40 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { rpc, rpcAsync } from "./rpcClient"; + +afterEach(() => vi.unstubAllGlobals()); + +const reason = "REFERENCE_INQUIRY_NOT_FOUND"; +const notice = `연구의 중간 기록은 저장됐습니다. 기준(reference) 단계만 실패했습니다 (사유: ${reason}). ` + + "결과는 기록에서 볼 수 있습니다. 같은 질문을 새로 보내면 연구가 다시 실행될 수 있으니 먼저 저장된 결과를 확인하세요. / " + + `Intermediate research records were saved. Only the reference step failed (reason: ${reason}). ` + + "You can read the result in the history. Sending the same question again as a new request may run the research again, so check the saved result first."; + +it("preserves the bilingual reference notice and saved-record metadata from an RPC error", async () => { + const data = { reason_code: reason, research_saved: true }; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ + jsonrpc: "2.0", id: "notice", error: { code: -32030, message: notice, data }, + }))); + await expect(rpc("thread/input", { project_id: "p" }, "reference-notice")).rejects.toMatchObject({ + name: "RpcError", code: -32030, message: `${notice} [-32030]`, details: data, + }); +}); + +it("shows the saved asynchronous operation's reference notice without replacing it", async () => { + const fetch = vi.fn() + .mockResolvedValueOnce(Response.json({ + jsonrpc: "2.0", id: "started", + result: { operation_id: "op:notice", state: "RUNNING", value: { status: "ACCEPTED_RUNNING" } }, + })) + .mockResolvedValueOnce(Response.json({ + operation_id: "op:notice", state: "FAILED", result: null, + error: { code: -32603, message: notice, data: { reason_code: reason, research_saved: true } }, + })); + vi.stubGlobal("fetch", fetch); + await expect(rpcAsync("thread/input", { project_id: "p", contract_version: 2 }, "async-notice", { + pollMs: 0, + })).rejects.toMatchObject({ message: notice }); + expect(fetch).toHaveBeenCalledTimes(2); + expect(fetch).toHaveBeenLastCalledWith("/operations/op%3Anotice?project_id=p", { + credentials: "same-origin", + }); +}); diff --git a/config/module-responsibility-budget.json b/config/module-responsibility-budget.json index 4a6d579..4ffacbf 100644 --- a/config/module-responsibility-budget.json +++ b/config/module-responsibility-budget.json @@ -123,7 +123,7 @@ "policy": "over the 600-line limit when it was introduced; do not grow it; lower the baseline when it is split" }, "src/thoth/application/commands/memory_full.py": { - "baseline_lines": 519, + "baseline_lines": 513, "top_level_symbols": [ "MemoryFullHandlers" ], diff --git a/docs/KNOWN_LIMITATIONS.ko.md b/docs/KNOWN_LIMITATIONS.ko.md index 7c4f2c7..5537ad2 100644 --- a/docs/KNOWN_LIMITATIONS.ko.md +++ b/docs/KNOWN_LIMITATIONS.ko.md @@ -38,11 +38,24 @@ handler를 다시 실행하거나 과거 요청을 자동으로 훑어 닫지는 있고, 새로 읽은 개정 번호로 보내면 같은 변경이 한 번 더 적용될 수도 있습니다. 이 복구가 외부 효과의 정확히 한 번 실행을 보장하는 것은 아닙니다. -## 끝난 요청의 후속 활동 기록이 빠질 수 있습니다 +## 같은 요청 재전송으로 빠진 성공 일지를 보충할 수 있습니다 -요청이 끝나면 결과를 먼저 저장하고, 활동 기록의 checkpoint·알림·"성공" 항목은 그 뒤에 저장합니다. -둘 사이에 저장소 오류나 종료가 겹치면 결과와 연결 정보는 그대로이고 같은 요청은 그 결과를 -돌려주지만, 후속 항목은 빠질 수 있고 나중에 채워지지 않습니다. 요청을 다시 할 필요는 없습니다. +로컬 모드에서는 실측한 20개 명령과 지원되는 저장 결과 형식에 한해, 같은 멱등 key를 다시 보내면 +빠진 checkpoint·알림·성공 기록을 채울 수 있습니다. 저장된 결과와 연결 정보는 그대로이며 +handler·모델·그 밖의 효과를 다시 실행하지 않습니다. 기록이 다 채워지면 재전송해도 추가하지 않습니다. + +새 항목의 시각은 실제 보충 시각이며, `supplemented: true`와 원 완료 시각을 담은 +`supplemented_for`가 붙습니다. 기존 항목은 고치지 않습니다. 보충 중 중단되면 그 거래의 새 기록은 +함께 취소되고 다음 같은 요청에서 다시 시도할 수 있습니다. + +심사용 모드, 실패·진행 중 요청, 연구 게시와 복원 게시에는 적용하지 않습니다. 다른 명령과 +확인하지 못한 옛 저장 형식도 보류합니다. 일지 형식이 불명확하거나 서로 충돌하는 경우, 시계 역행 +등으로 이벤트 체인의 정렬을 유지할 수 없는 경우, 작업 공간 소유권이 없거나 일시적인 보충 저장 +오류가 있는 경우에는 누락이 남을 수 있습니다. 기존 접근 권한 거절과 결과·binding 검증은 유지합니다. + +미측정 명령과 추가 source 분기 검토는 후속 작업입니다. 심사용·여러 프로세스 간 보충은 이번 변경 +범위 밖입니다. 일지만 고치려고 새 요청으로 다시 보내면 효과가 다시 실행될 수 있으니 저장된 결과를 +먼저 확인하세요. ## 샌드박스 연구 시험은 일부만 기록될 수 있습니다 diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 5f4681f..a2c3a9e 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -40,13 +40,26 @@ its ordinary change notification is omitted. Check the saved result before sendi A new question may use model capacity again; a newly refreshed revision guard can also permit a second change. This recovery is not a guarantee of exactly-once external effects. -## A finished request can miss some follow-up log entries - -When a request finishes, its result is saved first. The checkpoint, notification and "succeeded" -entries of the activity log are saved afterwards. If a storage error or shutdown hits between the -two, the result and its stored links stay intact and the same request returns that result, but -those follow-up entries can be missing and are not filled in later. You do not need to repeat the -request. +## Missing success-log entries can be supplemented on retry + +In LOCAL mode, retrying the same idempotency key can fill missing checkpoint, notification and +success entries for 20 measured commands and their supported saved result forms. The stored +result and its links remain unchanged. The handler, model and other effects are not run again; +once the missing entries are present, further retries add nothing. + +New entries use the actual supplementation time and carry `supplemented: true` and +`supplemented_for` with the original completion time. Existing entries are not rewritten. +An interrupted supplementation transaction rolls back its new entries and can be retried. + +This does not supplement hosted review, failed or running requests, research publication, or +restore publication. Other methods and unmeasured legacy result forms remain deferred. Unknown +or conflicting log formats, unsafe event-chain ordering (including clock rollback), missing +workspace ownership or temporary repair-storage failure can leave the gap in place. Existing +access denial and canonical result/binding checks still apply. + +Follow-up work includes the unmeasured methods and additional source-branch audits; hosted and +cross-process supplementation are outside this change. Do not send a new request just to repair +a log: it could run the effect again. Check the saved result first. ## Sandboxed research tests can leave a partial record diff --git a/src/thoth/adapters/storage/action.py b/src/thoth/adapters/storage/action.py index c576f0d..21a5f83 100644 --- a/src/thoth/adapters/storage/action.py +++ b/src/thoth/adapters/storage/action.py @@ -92,7 +92,7 @@ def list_actions(self, project_id: str) -> tuple[ActionRecord, ...]: rows = connection.execute( select(action_records) .where(action_records.c.project_id == project_id) - .order_by(action_records.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple(ActionRecord.model_validate(load(row["content_json"])) for row in rows) latest: dict[str, ActionRecord] = {} @@ -122,9 +122,7 @@ def read_action( action_records.c.action_id == action_id, ) if revision_digest is None: - statement = statement.order_by( - action_records.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(action_records.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -158,7 +156,7 @@ def list_portfolios(self, project_id: str) -> tuple[ActionPortfolioRecord, ...]: rows = connection.execute( select(action_portfolios) .where(action_portfolios.c.project_id == project_id) - .order_by(action_portfolios.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( ActionPortfolioRecord.model_validate(load(row["content_json"])) for row in rows @@ -190,9 +188,7 @@ def read_portfolio( action_portfolios.c.portfolio_id == portfolio_id, ) if revision_digest is None: - statement = statement.order_by( - action_portfolios.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(action_portfolios.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -228,7 +224,7 @@ def list_plans(self, project_id: str) -> tuple[ActionPlanRecord, ...]: rows = connection.execute( select(action_plans) .where(action_plans.c.project_id == project_id) - .order_by(action_plans.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( ActionPlanRecord.model_validate(load(row["content_json"])) for row in rows @@ -260,9 +256,7 @@ def read_plan( action_plans.c.plan_id == plan_id, ) if revision_digest is None: - statement = statement.order_by( - action_plans.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(action_plans.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: diff --git a/src/thoth/adapters/storage/control_record.py b/src/thoth/adapters/storage/control_record.py index dd0feb6..b3bf9eb 100644 --- a/src/thoth/adapters/storage/control_record.py +++ b/src/thoth/adapters/storage/control_record.py @@ -4,7 +4,7 @@ import orjson from pydantic import ValidationError -from sqlalchemy import Engine, insert, select +from sqlalchemy import Engine, Integer, insert, literal_column, select from thoth.adapters.storage.schema import control_records from thoth.adapters.storage.transaction import read_connection, write_connection @@ -55,7 +55,9 @@ def read(self, project_id: str, namespace: str, record_id: str) -> ControlRecord control_records.c.namespace == namespace, control_records.c.record_id == record_id, ) - .order_by(control_records.c.version.desc()) + .order_by( + control_records.c.version.desc(), literal_column("rowid", Integer()).desc() + ) .limit(1) ) .mappings() @@ -91,10 +93,13 @@ def list( ) if record_type is not None: statement = statement.where(control_records.c.record_type == record_type) + order = ( + (control_records.c.version, literal_column("rowid", Integer())) + if latest_only + else (control_records.c.created_at, control_records.c.version) + ) with read_connection(self._engine) as connection: - rows = connection.execute( - statement.order_by(control_records.c.created_at, control_records.c.version) - ).mappings() + rows = connection.execute(statement.order_by(*order)).mappings() parsed: list[ControlRecord] = [] for row in rows: try: diff --git a/src/thoth/adapters/storage/criterion_contract.py b/src/thoth/adapters/storage/criterion_contract.py index 5866e71..16901fe 100644 --- a/src/thoth/adapters/storage/criterion_contract.py +++ b/src/thoth/adapters/storage/criterion_contract.py @@ -140,7 +140,7 @@ def list_contracts(self, project_id: str) -> tuple[CriterionContractRecord, ...] rows = connection.execute( select(criterion_contracts) .where(criterion_contracts.c.project_id == project_id) - .order_by(criterion_contracts.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() return self._latest(tuple(self._contract(row) for row in rows)) @@ -168,9 +168,7 @@ def read_contract( if revision_digest is not None: statement = statement.where(criterion_contracts.c.revision_digest == revision_digest) else: - statement = statement.order_by( - criterion_contracts.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) with read_connection(self._engine) as connection: row = connection.execute(statement).mappings().first() return None if row is None else self._contract(row) diff --git a/src/thoth/adapters/storage/decision_object.py b/src/thoth/adapters/storage/decision_object.py index 26cde6f..e4b9b74 100644 --- a/src/thoth/adapters/storage/decision_object.py +++ b/src/thoth/adapters/storage/decision_object.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, insert, select +from sqlalchemy import Engine, Integer, insert, literal_column, select from sqlalchemy.dialects.sqlite import insert as sqlite_insert from thoth.adapters.storage.schema import ( @@ -186,7 +186,7 @@ def list_objects(self, project_id: str) -> tuple[DecisionObjectRecord, ...]: rows = connection.execute( select(decision_objects) .where(decision_objects.c.project_id == project_id) - .order_by(decision_objects.c.created_at) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( DecisionObjectRecord.model_validate(load(row["content_json"])) for row in rows @@ -209,7 +209,7 @@ def read_object( decision_objects.c.object_id == object_id, ) if revision_digest is None: - statement = statement.order_by(decision_objects.c.created_at.desc()).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(decision_objects.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -252,7 +252,7 @@ def list_relations(self, project_id: str, object_id: str) -> tuple[ObjectRelatio object_relations.c.project_id == project_id, object_relations.c.source_object_id == object_id, ) - .order_by(object_relations.c.created_at) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( ObjectRelationRecord.model_validate(load(row["content_json"])) for row in rows @@ -272,7 +272,7 @@ def read_relation(self, relation_id: str) -> ObjectRelationRecord | None: connection.execute( select(object_relations) .where(object_relations.c.relation_id == relation_id) - .order_by(object_relations.c.created_at.desc()) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() @@ -315,7 +315,9 @@ def list_attention( if object_id is not None: statement = statement.where(object_attention.c.object_id == object_id) with read_connection(self._engine) as connection: - rows = connection.execute(statement.order_by(object_attention.c.created_at)).mappings() + rows = connection.execute( + statement.order_by(literal_column("rowid", Integer())) + ).mappings() values = tuple( ObjectAttentionRecord.model_validate(load(row["content_json"])) for row in rows ) @@ -334,7 +336,7 @@ def read_attention(self, attention_id: str) -> ObjectAttentionRecord | None: connection.execute( select(object_attention) .where(object_attention.c.attention_id == attention_id) - .order_by(object_attention.c.created_at.desc()) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/events.py b/src/thoth/adapters/storage/events.py index 480a460..d1120f3 100644 --- a/src/thoth/adapters/storage/events.py +++ b/src/thoth/adapters/storage/events.py @@ -1,15 +1,20 @@ from __future__ import annotations +from collections.abc import Generator +from contextlib import contextmanager from datetime import datetime from typing import cast import orjson -from sqlalchemy import Connection, Engine, insert, select +from sqlalchemy import Connection, Engine, Integer, insert, literal_column, select +from sqlalchemy.exc import OperationalError from thoth.adapters.storage.schema import checkpoints, events from thoth.adapters.storage.transaction import read_connection, write_connection from thoth.domain.event import Checkpoint, EventRecord from thoth.ports.event_store import EventStorePort +from thoth.ports.journal import JournalCompletionUnavailable +from thoth.ports.runtime import AtomicUnitOfWorkPort def _object(value: str) -> dict[str, object]: @@ -40,6 +45,36 @@ class SqliteEventStore(EventStorePort): def __init__(self, engine: Engine) -> None: self._engine = engine + @contextmanager + def completion_transaction(self, uow: AtomicUnitOfWorkPort) -> Generator[None]: + try: + with uow.transaction(): + yield + except OperationalError as exc: + raise JournalCompletionUnavailable("JOURNAL_STORAGE_UNAVAILABLE") from exc + + def list_checkpoints(self, operation_id: str) -> tuple[Checkpoint, ...]: + with read_connection(self._engine) as connection: + rows = ( + connection.execute( + select(checkpoints) + .where(checkpoints.c.operation_id == operation_id) + .order_by(checkpoints.c.created_at, checkpoints.c.checkpoint_id) + ) + .mappings() + .all() + ) + return tuple( + Checkpoint( + checkpoint_id=str(row["checkpoint_id"]), + operation_id=str(row["operation_id"]), + payload=_object(str(row["payload_json"])), + checkpoint_digest=str(row["checkpoint_digest"]), + created_at=datetime.fromisoformat(str(row["created_at"])), + ) + for row in rows + ) + def insert_event(self, event: EventRecord) -> None: with write_connection(self._engine) as connection: insert_event_on_connection(connection, event) @@ -93,7 +128,7 @@ def latest_checkpoint(self, operation_id: str) -> Checkpoint | None: connection.execute( select(checkpoints) .where(checkpoints.c.operation_id == operation_id) - .order_by(checkpoints.c.created_at.desc(), checkpoints.c.checkpoint_id.desc()) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/evidence_graph.py b/src/thoth/adapters/storage/evidence_graph.py index 7b66489..bde080f 100644 --- a/src/thoth/adapters/storage/evidence_graph.py +++ b/src/thoth/adapters/storage/evidence_graph.py @@ -93,10 +93,7 @@ def read_source_by_artifact(self, artifact_id: str) -> EvidenceSourceRecord | No connection.execute( select(evidence_sources) .where(evidence_sources.c.artifact_id == artifact_id) - .order_by( - evidence_sources.c.created_at.desc(), - literal_column("rowid", Integer()).desc(), - ) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/execution.py b/src/thoth/adapters/storage/execution.py index 3afd8d5..7f87ab3 100644 --- a/src/thoth/adapters/storage/execution.py +++ b/src/thoth/adapters/storage/execution.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, Integer, Table, insert, literal_column, select +from sqlalchemy import Engine, Integer, Table, func, insert, literal_column, select from thoth.adapters.storage.schema import ( execution_audit, @@ -74,7 +74,12 @@ def list_executions(self, project_id: str) -> tuple[PlanExecutionRecord, ...]: rows = connection.execute( select(plan_executions) .where(plan_executions.c.project_id == project_id) - .order_by(plan_executions.c.created_at, literal_column("rowid", Integer())) + .order_by( + func.coalesce( + func.json_extract(plan_executions.c.content_json, "$.revision"), 0 + ), + literal_column("rowid", Integer()), + ) ).mappings() values = tuple( PlanExecutionRecord.model_validate(load(row["content_json"])) for row in rows @@ -94,7 +99,9 @@ def read_execution(self, project_id: str, plan_execution_id: str) -> PlanExecuti plan_executions.c.plan_execution_id == plan_execution_id, ) .order_by( - plan_executions.c.created_at.desc(), + func.coalesce( + func.json_extract(plan_executions.c.content_json, "$.revision"), 0 + ).desc(), literal_column("rowid", Integer()).desc(), ) .limit(1) @@ -128,7 +135,7 @@ def list_attempts( step_execution_attempts.c.project_id == project_id, step_execution_attempts.c.plan_execution_id == plan_execution_id, ) - .order_by(step_execution_attempts.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( StepExecutionAttemptRecord.model_validate(load(row["content_json"])) for row in rows @@ -147,10 +154,7 @@ def read_attempt(self, project_id: str, attempt_id: str) -> StepExecutionAttempt step_execution_attempts.c.project_id == project_id, step_execution_attempts.c.attempt_id == attempt_id, ) - .order_by( - step_execution_attempts.c.created_at.desc(), - literal_column("rowid", Integer()).desc(), - ) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() diff --git a/src/thoth/adapters/storage/hypothesis.py b/src/thoth/adapters/storage/hypothesis.py index 0e349a0..cd4a2dd 100644 --- a/src/thoth/adapters/storage/hypothesis.py +++ b/src/thoth/adapters/storage/hypothesis.py @@ -116,9 +116,7 @@ def read_hypothesis( hypothesis_records.c.hypothesis_id == hypothesis_id, ) if revision_digest is None: - statement = statement.order_by( - hypothesis_records.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(hypothesis_records.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -170,9 +168,7 @@ def read_portfolio( hypothesis_portfolios.c.portfolio_id == portfolio_id, ) if revision_digest is None: - statement = statement.order_by( - hypothesis_portfolios.c.created_at.desc(), literal_column("rowid", Integer()).desc() - ).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(hypothesis_portfolios.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -199,7 +195,7 @@ def list_relations( ) -> tuple[HypothesisRelationRecord, ...]: statement = ( select(hypothesis_relations) - .order_by(hypothesis_relations.c.created_at, literal_column("rowid", Integer())) + .order_by(literal_column("rowid", Integer())) .where(hypothesis_relations.c.project_id == project_id) ) if portfolio_id is not None: @@ -216,10 +212,7 @@ def read_relation(self, relation_id: str) -> HypothesisRelationRecord | None: connection.execute( select(hypothesis_relations) .where(hypothesis_relations.c.relation_id == relation_id) - .order_by( - hypothesis_relations.c.created_at.desc(), - literal_column("rowid", Integer()).desc(), - ) + .order_by(literal_column("rowid", Integer()).desc()) .limit(1) ) .mappings() @@ -398,7 +391,7 @@ def _list[T: DomainModel]( return self._execute( select(table) .where(table.c[field] == value) - .order_by(table.c.created_at, literal_column("rowid", Integer())), + .order_by(literal_column("rowid", Integer())), model, ) diff --git a/src/thoth/adapters/storage/outcome.py b/src/thoth/adapters/storage/outcome.py index cb2dd95..290c491 100644 --- a/src/thoth/adapters/storage/outcome.py +++ b/src/thoth/adapters/storage/outcome.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, Table, insert, select +from sqlalchemy import Engine, Integer, Table, func, insert, literal_column, select from sqlalchemy.dialects.sqlite import insert as sqlite_insert from thoth.adapters.storage.schema import ( @@ -115,7 +115,12 @@ def list_series(self, project_id: str) -> tuple[OutcomeSeriesRecord, ...]: rows = connection.execute( select(outcome_series) .where(outcome_series.c.project_id == project_id) - .order_by(outcome_series.c.created_at) + .order_by( + func.coalesce( + func.json_extract(outcome_series.c.content_json, "$.revision"), 0 + ), + literal_column("rowid", Integer()), + ) ).mappings() values = tuple( OutcomeSeriesRecord.model_validate(load(row["content_json"])) for row in rows @@ -133,7 +138,12 @@ def read_series( outcome_series.c.outcome_series_id == series_id, ) if revision_digest is None: - statement = statement.order_by(outcome_series.c.created_at.desc()).limit(1) + statement = statement.order_by( + func.coalesce( + func.json_extract(outcome_series.c.content_json, "$.revision"), 0 + ).desc(), + literal_column("rowid", Integer()).desc(), + ).limit(1) else: statement = statement.where(outcome_series.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: @@ -159,7 +169,7 @@ def list_assessments(self, project_id: str) -> tuple[OutcomeAssessmentRecord, .. rows = connection.execute( select(outcome_assessments) .where(outcome_assessments.c.project_id == project_id) - .order_by(outcome_assessments.c.created_at) + .order_by(literal_column("rowid", Integer())) ).mappings() values = tuple( OutcomeAssessmentRecord.model_validate(load(row["content_json"])) for row in rows @@ -177,7 +187,7 @@ def read_assessment( outcome_assessments.c.outcome_assessment_id == assessment_id, ) if revision_digest is None: - statement = statement.order_by(outcome_assessments.c.created_at.desc()).limit(1) + statement = statement.order_by(literal_column("rowid", Integer()).desc()).limit(1) else: statement = statement.where(outcome_assessments.c.revision_digest == revision_digest) with read_connection(self._engine) as connection: diff --git a/src/thoth/adapters/storage/thread_runtime.py b/src/thoth/adapters/storage/thread_runtime.py index e41df81..d482b6f 100644 --- a/src/thoth/adapters/storage/thread_runtime.py +++ b/src/thoth/adapters/storage/thread_runtime.py @@ -4,7 +4,7 @@ from typing import cast import orjson -from sqlalchemy import Engine, func, insert, select, update +from sqlalchemy import Engine, Integer, func, insert, literal_column, select, update from thoth.adapters.storage.schema import thread_activities, thread_checkpoints, thread_inputs from thoth.adapters.storage.transaction import read_connection, write_connection @@ -87,7 +87,7 @@ def list_checkpoints(self, project_id: str, thread_id: str) -> tuple[ThreadCheck thread_checkpoints.c.project_id == project_id, thread_checkpoints.c.thread_id == thread_id, ) - .order_by(thread_checkpoints.c.created_at, thread_checkpoints.c.checkpoint_id) + .order_by(literal_column("rowid", Integer())) ).mappings() return tuple(self._checkpoint(row) for row in rows) diff --git a/src/thoth/adapters/storage/workspace_owner.py b/src/thoth/adapters/storage/workspace_owner.py index 5a38d08..372d20e 100644 --- a/src/thoth/adapters/storage/workspace_owner.py +++ b/src/thoth/adapters/storage/workspace_owner.py @@ -190,6 +190,11 @@ def holds_workspace(workspace: Path) -> bool: return _key(workspace) in _OWNED +def workspace_owner_key(workspace: Path) -> str: + """Use the existing ownership identity for other process-local coordination.""" + return _key(workspace) + + @contextmanager def pin_workspace_owner(workspace: Path) -> Generator[bool]: """Retain only an already-owned lock; never acquire or create a lock file.""" diff --git a/src/thoth/application/commands/memory_full.py b/src/thoth/application/commands/memory_full.py index 37700b3..a0e717c 100644 --- a/src/thoth/application/commands/memory_full.py +++ b/src/thoth/application/commands/memory_full.py @@ -1,5 +1,6 @@ from __future__ import annotations +from functools import partial from typing import cast from pydantic import JsonValue @@ -42,6 +43,7 @@ projection_status_view, select_policies, ) +from thoth.application.commands.memory_records import memory_record from thoth.application.services.control_record_service import ControlRecordService from thoth.application.services.memory_router import build_recall_context from thoth.domain.control_record import ControlRecord @@ -63,6 +65,7 @@ def __init__( ) -> None: self._records = records self._controls = controls + self._memory_record = partial(memory_record, controls) self._legacy = legacy self._full = full self._ledger = ledger @@ -303,23 +306,19 @@ async def candidate_classify(self, value: dict[str, JsonValue]) -> dict[str, Jso async def validate(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: request = ValidateInput.model_validate(value) - current = self._candidate(request) - plan, reducer, verdicts, gate_payload, gate_digest = gate_draft(request, current) - gate = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="GATE", - state=reducer, - payload={**gate_payload, "gate_digest": gate_digest}, - ) - revised = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="CANDIDATE", - record_id=current.record_id, - state=reducer, - payload={**current.payload, "validated_gate_digest": gate_digest}, - ) + with self._ledger.transaction(): + current = self._candidate(request) + plan, reducer, verdicts, gate_payload, gate_digest = gate_draft(request, current) + gate = self._memory_record( + request, "GATE", reducer, {**gate_payload, "gate_digest": gate_digest} + ) + revised = self._memory_record( + request, + "CANDIDATE", + reducer, + {**current.payload, "validated_gate_digest": gate_digest}, + record_id=current.record_id, + ) return cast( dict[str, JsonValue], { @@ -334,34 +333,30 @@ async def validate(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: async def change_set_propose(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: request = ChangeSetProposeInput.model_validate(value) - candidate = self._candidate(request) - if ( - candidate.state != "VALIDATED" - or candidate.payload.get("validated_gate_digest") != request.validated_gate_digest - ): - raise RpcApplicationError( - RpcErrorCode.DOMAIN_REJECTED, "candidate gate is not validated" + with self._ledger.transaction(): + candidate = self._candidate(request) + if ( + candidate.state != "VALIDATED" + or candidate.payload.get("validated_gate_digest") != request.validated_gate_digest + ): + raise RpcApplicationError( + RpcErrorCode.DOMAIN_REJECTED, "candidate gate is not validated" + ) + entry_payload = memory_entry_payload(candidate) + memory_candidate = self._memory_record( + request, "MEMORY_ENTRY_CANDIDATE", "PROPOSED_NOT_APPLIED", entry_payload + ) + change_set = self._memory_record( + request, + "CHANGE_SET_PROPOSAL", + "PROPOSED_NOT_APPLIED", + { + "memory_candidate_digest": memory_candidate.record_digest, + "transition_intent": request.transition_intent, + "expected_memory_head_digest": request.expected_memory_head_digest, + "current_state_mutated": False, + }, ) - entry_payload = memory_entry_payload(candidate) - memory_candidate = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="MEMORY_ENTRY_CANDIDATE", - state="PROPOSED_NOT_APPLIED", - payload=entry_payload, - ) - change_set = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="CHANGE_SET_PROPOSAL", - state="PROPOSED_NOT_APPLIED", - payload={ - "memory_candidate_digest": memory_candidate.record_digest, - "transition_intent": request.transition_intent, - "expected_memory_head_digest": request.expected_memory_head_digest, - "current_state_mutated": False, - }, - ) return cast( dict[str, JsonValue], { @@ -405,28 +400,27 @@ async def revalidate(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: async def retire_propose(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: request = RetireProposeInput.model_validate(value) - current = self._memory(request) - if current.record_digest != request.expected_revision_digest: - raise RpcApplicationError(RpcErrorCode.DOMAIN_REJECTED, "memory revision changed") - candidate = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="RETIRE_CANDIDATE", - state="PROPOSED_NOT_APPLIED", - payload={ - "memory_entry_id": current.record_id, - "reason": request.reason, - "evidence_refs": request.evidence_refs, - "history_preserved": True, - }, - ) - change_set = self._controls.create( - project_id=request.project_id, - namespace="MEMORY", - record_type="CHANGE_SET_PROPOSAL", - state="PROPOSED_NOT_APPLIED", - payload={"retire_candidate_digest": candidate.record_digest}, - ) + with self._ledger.transaction(): + current = self._memory(request) + if current.record_digest != request.expected_revision_digest: + raise RpcApplicationError(RpcErrorCode.DOMAIN_REJECTED, "memory revision changed") + candidate = self._memory_record( + request, + "RETIRE_CANDIDATE", + "PROPOSED_NOT_APPLIED", + { + "memory_entry_id": current.record_id, + "reason": request.reason, + "evidence_refs": request.evidence_refs, + "history_preserved": True, + }, + ) + change_set = self._memory_record( + request, + "CHANGE_SET_PROPOSAL", + "PROPOSED_NOT_APPLIED", + {"retire_candidate_digest": candidate.record_digest}, + ) return { "retirement_candidate": candidate.model_dump(mode="json"), "change_set": change_set.model_dump(mode="json"), diff --git a/src/thoth/application/commands/memory_records.py b/src/thoth/application/commands/memory_records.py new file mode 100644 index 0000000..3f87232 --- /dev/null +++ b/src/thoth/application/commands/memory_records.py @@ -0,0 +1,24 @@ +"""Typed MEMORY record creation; callers own transaction and execution order.""" + +from thoth.application.commands.memory_inputs import ProjectInput +from thoth.application.services.control_record_service import ControlRecordService +from thoth.domain.control_record import ControlRecord + + +def memory_record( + controls: ControlRecordService, + request: ProjectInput, + kind: str, + state: str, + payload: dict[str, object], + *, + record_id: str | None = None, +) -> ControlRecord: + return controls.create( + project_id=request.project_id, + namespace="MEMORY", + record_type=kind, + state=state, + payload=payload, + record_id=record_id, + ) diff --git a/src/thoth/application/commands/reference.py b/src/thoth/application/commands/reference.py index c63da97..b27d3af 100644 --- a/src/thoth/application/commands/reference.py +++ b/src/thoth/application/commands/reference.py @@ -3,19 +3,26 @@ from pydantic import JsonValue from thoth.application.commands.thread_analysis import ThreadInputRequest +from thoth.application.services.reference_failure import reference_failure from thoth.application.services.reference_mapping import ModelReferenceMapper from thoth.application.services.reference_service import ReferenceService, ReferenceTransition from thoth.domain.evidence import InformationSufficiencyAssessment from thoth.domain.reference import ReferenceAnswerRequest, ReferenceError, ReferenceRequest +from thoth.ports.ledger import LedgerPort from thoth.ports.thread import ThreadEntryPort from thoth.protocol.jsonrpc import RpcApplicationError, RpcErrorCode class ReferenceThreadEntry: def __init__( - self, delegate: ThreadEntryPort, service: ReferenceService, mapper: ModelReferenceMapper + self, + delegate: ThreadEntryPort, + service: ReferenceService, + mapper: ModelReferenceMapper, + ledger: LedgerPort, ) -> None: self._delegate, self._service, self._mapper = delegate, service, mapper + self._ledger = ledger def authorize_before_claim(self, method: str, value: dict[str, JsonValue]) -> None: self._delegate.authorize_before_claim(method, value) @@ -45,13 +52,18 @@ async def analyze(self, value: dict[str, JsonValue]) -> dict[str, JsonValue]: selected.expected_revision_digest, ) result = await self._delegate.analyze(value) - if result.get("queued") is True or "assessment" not in result: - return result - transition = await self._reference(request, result) except ReferenceError as exc: raise RpcApplicationError( RpcErrorCode.DOMAIN_REJECTED, exc.code, data={"reason_code": exc.code} ) from exc + if result.get("queued") is True or "assessment" not in result: + return result + try: + transition = await self._reference(request, result) + except ReferenceError as exc: + raise reference_failure( + self._ledger, request.project_id, request.thread_id, result, exc.code + ) from exc if transition is None: return result return { diff --git a/src/thoth/application/commands/threads.py b/src/thoth/application/commands/threads.py index 072bb35..2a3bf20 100644 --- a/src/thoth/application/commands/threads.py +++ b/src/thoth/application/commands/threads.py @@ -247,7 +247,10 @@ async def checkpoint_list(self, value: dict[str, JsonValue]) -> dict[str, JsonVa return { "checkpoints": [ item.model_dump(mode="json") - for item in self._runtime.list_checkpoints(request.project_id, request.thread_id) + for item in sorted( + self._runtime.list_checkpoints(request.project_id, request.thread_id), + key=lambda item: (item.created_at.isoformat(), item.checkpoint_id), + ) ] } diff --git a/src/thoth/application/services/r2_closed_loop.py b/src/thoth/application/services/r2_closed_loop.py index 10f67da..28071a6 100644 --- a/src/thoth/application/services/r2_closed_loop.py +++ b/src/thoth/application/services/r2_closed_loop.py @@ -15,6 +15,10 @@ stale_head_projection, template_required_projection, ) +from thoth.application.services.r2_failure_publication import ( + AdmissionRejected, + R2FailurePublication, +) from thoth.application.services.r2_outcome_projection import ( ProcessOutcomeProjection, build_process_outcome, @@ -24,11 +28,8 @@ closed_loop_actor, failure_class_of, r2_action, - reconciliation_record, - recovery_actor, recovery_attempt, - recovery_candidate, - repair_basis_text, + recovery_draft, stage_revision, terminal_state, ) @@ -114,6 +115,7 @@ def __init__( self._clock = clock self._ids = ids self._test_lifecycle = test_lifecycle + self._failures = R2FailurePublication(ledger, sandbox) async def execute_if_required( self, @@ -152,71 +154,81 @@ async def execute_if_required( return run_hold assert bundle is not None if head_set_digest(self._ledger.read_heads(project.project_id)) != sealed_head_digest: - self._record_test_hold(prepared_test, bundle, "SANDBOX_HEAD_STALE_AFTER_IO") - return self._stale_head_execution( - project.project_id, - portfolio, - action_plan, - action.action_id, - bundle, - ) + with self._failures.transaction(): + self._record_test_hold(prepared_test, bundle, "SANDBOX_HEAD_STALE_AFTER_IO") + return self._stale_head_execution( + project.project_id, + portfolio, + action_plan, + action.action_id, + bundle, + ) failure_class = failure_class_of(bundle, recovery_policy) assessments, assessment_hold = self._measurement_assessments(prepared_test, bundle) if assessment_hold is not None: return assessment_hold - if failure_class is not None: - try: - with self._sandbox.accepting_observation(spec, bundle): - if prepared_test is not None and self._test_lifecycle is not None: - failure_completion = self._test_lifecycle.complete( - prepared_test, bundle, assessments - ) - except SandboxFailure as exc: - self._record_test_hold(prepared_test, bundle, exc.code.value) - return self._admission_hold( - project.project_id, portfolio, action_plan, action.action_id, bundle, exc - ) - attempts.append(recovery_attempt(spec, bundle, failure_class, retry_ordinal)) - if failure_class is None: - break - if ( + retry_allowed = ( failure_class == ExecutionFailureClass.TRANSIENT_INFRA and retry_ordinal < recovery_policy.max_transient_retries - ): - retry_ordinal += 1 - if prepared_test is not None and self._test_lifecycle is not None: - try: - prepared_test = self._test_lifecycle.retry(project, action, prepared_test) - except ValueError as exc: - return self._test_hold(exc, bundle=bundle) - spec, portfolio, action_plan = ( - prepared_test.spec, - prepared_test.portfolio, - prepared_test.action_plan, - ) - sealed_heads = dict(self._ledger.read_heads(project.project_id)) - sealed_head_digest = head_set_digest(sealed_heads) - research = self._research.context( - project.project_id, portfolio.object_id, sealed_heads - ) - else: - spec = spec.model_copy(update={"attempt_id": self._ids.new("r2-attempt")}) - continue - return self._failure_execution( - project=project, - action_plan=action_plan, - action_id=action.action_id, - spec=spec, - failure_class=failure_class, - attempts=tuple(attempts), - expected_heads=sealed_heads, - expected_head_set_digest=sealed_head_digest, - retry_budget_exhausted=( - failure_class == ExecutionFailureClass.TRANSIENT_INFRA - and retry_ordinal >= recovery_policy.max_transient_retries - ), - test_completion=failure_completion, ) + try: + with self._failures.final_attempt(prepared_test, failure_class, retry_allowed): + if failure_class is not None: + with self._failures.accepting_failure(spec, bundle): + if prepared_test is not None and self._test_lifecycle is not None: + failure_completion = self._test_lifecycle.complete( + prepared_test, bundle, assessments + ) + attempts.append(recovery_attempt(spec, bundle, failure_class, retry_ordinal)) + if failure_class is None: + break + if retry_allowed: + retry_ordinal += 1 + if prepared_test is not None and self._test_lifecycle is not None: + try: + prepared_test = self._test_lifecycle.retry( + project, action, prepared_test + ) + except ValueError as exc: + return self._test_hold(exc, bundle=bundle) + spec, portfolio, action_plan = ( + prepared_test.spec, + prepared_test.portfolio, + prepared_test.action_plan, + ) + sealed_heads = dict(self._ledger.read_heads(project.project_id)) + sealed_head_digest = head_set_digest(sealed_heads) + research = self._research.context( + project.project_id, portfolio.object_id, sealed_heads + ) + else: + spec = spec.model_copy( + update={"attempt_id": self._ids.new("r2-attempt")} + ) + continue + return self._failure_execution( + project=project, + action_plan=action_plan, + action_id=action.action_id, + spec=spec, + failure_class=failure_class, + attempts=tuple(attempts), + expected_heads=sealed_heads, + expected_head_set_digest=sealed_head_digest, + retry_budget_exhausted=( + failure_class == ExecutionFailureClass.TRANSIENT_INFRA + and retry_ordinal >= recovery_policy.max_transient_retries + ), + test_completion=failure_completion, + ) + except AdmissionRejected as rejected: + exc = rejected.failure + with self._failures.transaction(): + self._failures.reject_admission(bundle) + self._record_test_hold(prepared_test, bundle, exc.code.value) + return self._admission_hold( + project.project_id, portfolio, action_plan, action.action_id, bundle, exc + ) process = build_process_outcome( project=project, portfolio=portfolio, @@ -266,7 +278,9 @@ async def execute_if_required( assessments, ) except SandboxFailure as exc: - self._record_test_hold(prepared_test, bundle, exc.code.value) + with self._failures.transaction(): + self._failures.reject_admission(bundle) + self._record_test_hold(prepared_test, bundle, exc.code.value) return self._admission_hold( project.project_id, portfolio, action_plan, action.action_id, bundle, exc ) @@ -371,8 +385,9 @@ def _measurement_assessments( try: return self._test_lifecycle.prepare_assessments(prepared, bundle), None except ValueError as exc: - self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_ASSESSMENT_REJECTED") - self._record_test_hold(prepared, bundle, "RESEARCH_TEST_ASSESSMENT_REJECTED") + with self._failures.transaction(): + self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_ASSESSMENT_REJECTED") + self._record_test_hold(prepared, bundle, "RESEARCH_TEST_ASSESSMENT_REJECTED") return (), self._test_hold(exc, bundle=bundle) def _record_test_hold( @@ -424,7 +439,7 @@ def _commit_outcome( assessments: tuple[TestValidityAssessment, ...] = (), ) -> tuple[CommitResult, ResearchTestCompletion | None] | R2ClosedLoopExecution: try: - with self._sandbox.accepting_observation(spec, bundle): + with self._sandbox.accepting_observation(spec, bundle, defer_rejection=True): commit = RevisionCommitService( self._ledger, self._clock, @@ -458,8 +473,9 @@ def _commit_outcome( except Exception: if prepared_test is None: raise - self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_FINALIZATION_FAILED") - self._record_test_hold(prepared_test, bundle, "RESEARCH_TEST_FINALIZATION_FAILED") + with self._failures.transaction(): + self._sandbox.reject_observation(bundle, reason="RESEARCH_TEST_FINALIZATION_FAILED") + self._record_test_hold(prepared_test, bundle, "RESEARCH_TEST_FINALIZATION_FAILED") return self._test_hold(ValueError("RESEARCH_TEST_FINALIZATION_FAILED"), bundle=bundle) def _admission_hold( @@ -563,27 +579,12 @@ def _failure_execution( if test_completion is not None: expected_heads = dict(self._ledger.read_heads(project.project_id)) expected_head_set_digest = head_set_digest(expected_heads) - baseline_heads = dict(expected_heads) - baseline_digest = head_set_digest(baseline_heads) - repair_basis = repair_basis_text(failure_class, attempts) - candidate = recovery_candidate( - self._ids, - action_plan, - action_id, - baseline_digest, - failure_class, - repair_basis, - attempts, + draft = recovery_draft( + self._ids, action_plan, action_id, expected_heads, failure_class, attempts ) - reconciliation = None - if failure_class == ExecutionFailureClass.AMBIGUOUS_EXTERNAL: - reconciliation = reconciliation_record(self._ids, action_id, attempts) - revised_plan = action_plan.model_copy(update={"recovery_candidate": candidate}) - action_head_key = f"ACTION:{action_plan.plan_id}" - actor = recovery_actor() - research = self._research.context(project.project_id, action_plan.object_id, baseline_heads) + research = self._research.context(project.project_id, action_plan.object_id, draft.heads) batch = self._research.stage_actions( - revised_plan, actor=actor, context=research, hypotheses=research.hypotheses + draft.plan, actor=draft.actor, context=research, hypotheses=research.hypotheses ) # Recovery proposes a plan-only branch; no action/portfolio projection is activated. staged = batch.staged[0] @@ -596,27 +597,26 @@ def _failure_execution( RevisionChangeSet( changeset_id=self._ids.new("changeset"), project_id=project.project_id, - expected_heads={action_head_key: "0" * 64}, + expected_heads={draft.action_head_key: "0" * 64}, expected_head_set_digest=expected_head_set_digest, staged_revisions=(staged,), impact_plan=ImpactPropagationPlan(), - actor=actor, - reason=repair_basis, + actor=draft.actor, + reason=draft.repair_basis, ) ) baseline_after = head_set_digest(self._ledger.read_heads(project.project_id)) - terminal = terminal_state(failure_class, retry_budget_exhausted) return R2ClosedLoopExecution( projection=failure_projection( - terminal, + terminal_state(failure_class, retry_budget_exhausted), test_completion, action_id, attempts, - baseline_digest, + draft.digest, baseline_after, commit, - repair_basis, - reconciliation, + draft.repair_basis, + draft.reconciliation, retry_budget_exhausted, ) ) diff --git a/src/thoth/application/services/r2_failure_publication.py b/src/thoth/application/services/r2_failure_publication.py new file mode 100644 index 0000000..3245bb7 --- /dev/null +++ b/src/thoth/application/services/r2_failure_publication.py @@ -0,0 +1,56 @@ +"""Short failure publication boundaries, entered only after external sandbox work.""" + +from collections.abc import Generator +from contextlib import contextmanager + +from thoth.application.services.r2_test_lifecycle import PreparedResearchExecution +from thoth.application.services.sandbox_service import SandboxExecutionBundle, SandboxService +from thoth.domain.recovery import ExecutionFailureClass +from thoth.domain.sandbox import SandboxFailure, SandboxRunSpec +from thoth.ports.ledger import LedgerPort + + +class AdmissionRejected(Exception): + """Only an acceptance failure; a later recovery-candidate error is never wrapped.""" + + def __init__(self, failure: SandboxFailure) -> None: + super().__init__(str(failure)) + self.failure = failure + + +class R2FailurePublication: + def __init__(self, ledger: LedgerPort, sandbox: SandboxService) -> None: + self._ledger = ledger + self._sandbox = sandbox + + @contextmanager + def transaction(self) -> Generator[None]: + with self._ledger.transaction(): + yield + + @contextmanager + def final_attempt( + self, + prepared: PreparedResearchExecution | None, + failure_class: ExecutionFailureClass | None, + retry_allowed: bool, + ) -> Generator[None]: + if prepared is not None and failure_class is not None and not retry_allowed: + with self._ledger.transaction(): + yield + else: + # Plain execution, success and retry paths keep their existing boundaries. + yield + + @contextmanager + def accepting_failure( + self, spec: SandboxRunSpec, bundle: SandboxExecutionBundle + ) -> Generator[None]: + try: + with self._sandbox.accepting_observation(spec, bundle, defer_rejection=True): + yield + except SandboxFailure as failure: + raise AdmissionRejected(failure) from failure + + def reject_admission(self, bundle: SandboxExecutionBundle) -> None: + self._sandbox.reject_observation(bundle, reason="SANDBOX_ADMISSION_BASIS_CHANGED") diff --git a/src/thoth/application/services/r2_recovery_records.py b/src/thoth/application/services/r2_recovery_records.py index 23a8d20..5e753c4 100644 --- a/src/thoth/application/services/r2_recovery_records.py +++ b/src/thoth/application/services/r2_recovery_records.py @@ -3,19 +3,20 @@ These functions build the values the coordinator stages, links or reports: the R2 action pick, a recovery attempt, the linked outcome, the two coordinator actors, failure classification, the recovery candidate and reconciliation records, and the staged revision of one entity. They read -nothing and write nothing. The coordinator keeps every await, every ledger, sandbox, research and -lifecycle call and every transaction, and passes the id generator and the clock through as the -original objects so ids and times are drawn in the same order as before. +nothing and write nothing to storage. The coordinator and failure publisher retain every await, +ledger, sandbox, research, lifecycle call and transaction. The original id generator and clock +are passed through, so ids and times are drawn in the same order as before. """ from __future__ import annotations +from dataclasses import dataclass from typing import cast from thoth.application.services.sandbox_service import SandboxExecutionBundle from thoth.domain.action import ActionCandidate, ActionPlan from thoth.domain.actor import ActorRef -from thoth.domain.canonical import canonical_payload, domain_digest +from thoth.domain.canonical import canonical_payload, domain_digest, head_set_digest from thoth.domain.enums import ActorKind, EntityType, RiskTier from thoth.domain.outcome import OutcomeRecord from thoth.domain.policy import R2RecoveryPolicy @@ -197,6 +198,48 @@ def reconciliation_record( return reconciliation +@dataclass(frozen=True) +class RecoveryDraft: + heads: dict[str, str] + digest: str + repair_basis: str + plan: ActionPlan + reconciliation: ReconciliationRecord | None + action_head_key: str + actor: ActorRef + + +def recovery_draft( + ids: IdGeneratorPort, + action_plan: ActionPlan, + action_id: str, + expected_heads: dict[str, str], + failure_class: ExecutionFailureClass, + attempts: tuple[RecoveryAttempt, ...], +) -> RecoveryDraft: + baseline_heads = dict(expected_heads) + baseline_digest = head_set_digest(baseline_heads) + repair_basis = repair_basis_text(failure_class, attempts) + candidate = recovery_candidate( + ids, action_plan, action_id, baseline_digest, failure_class, repair_basis, attempts + ) + reconciliation = None + if failure_class == ExecutionFailureClass.AMBIGUOUS_EXTERNAL: + reconciliation = reconciliation_record(ids, action_id, attempts) + revised_plan = action_plan.model_copy(update={"recovery_candidate": candidate}) + action_head_key = f"ACTION:{action_plan.plan_id}" + actor = recovery_actor() + return RecoveryDraft( + baseline_heads, + baseline_digest, + repair_basis, + revised_plan, + reconciliation, + action_head_key, + actor, + ) + + def terminal_state(failure_class: ExecutionFailureClass, retry_budget_exhausted: bool) -> str: terminal = ( "RECONCILIATION_REQUIRED" diff --git a/src/thoth/application/services/reference_failure.py b/src/thoth/application/services/reference_failure.py new file mode 100644 index 0000000..4d02915 --- /dev/null +++ b/src/thoth/application/services/reference_failure.py @@ -0,0 +1,112 @@ +"""Trusted reference-only notices, after verifying the returned research commit is readable.""" + +import re + +from pydantic import JsonValue, ValidationError + +from thoth.application.services.revision_service import CommitResult +from thoth.domain.research_execution import research_work +from thoth.domain.research_failure import FailureOrigin, ResearchFailureCause, failure_cause +from thoth.ports.ledger import LedgerPort +from thoth.protocol.deferred import current_operation +from thoth.protocol.jsonrpc import RpcApplicationError, RpcErrorCode + + +class SavedReferenceFailure(RpcApplicationError): + def __init__(self, reason_code: str, *, intermediate: bool) -> None: + if re.fullmatch(r"REFERENCE_[A-Z0-9_]{1,149}", reason_code) is None: + raise ValueError("REFERENCE_NOTICE_REASON_INVALID") + self.reason_code = reason_code + ko = "연구의 중간 기록은 저장됐습니다." if intermediate else "연구 결과는 저장됐습니다." + en = ( + "Intermediate research records were saved." + if intermediate + else "The research result was saved." + ) + message = ( + f"{ko} 기준(reference) 단계만 실패했습니다 (사유: {reason_code}). " + "결과는 기록에서 볼 수 있습니다. 같은 질문을 새로 보내면 연구가 다시 실행될 수 있으니 " + "먼저 저장된 결과를 확인하세요. / " + f"{en} Only the reference step failed (reason: {reason_code}). " + "You can read the result in the history. Sending the same question again as a new " + "request may run the research again, so check the saved result first." + ) + super().__init__( + RpcErrorCode.DOMAIN_REJECTED, + message, + data={"reason_code": reason_code, "research_saved": True}, + ) + + +def reference_failure_cause( + error: Exception, origin: FailureOrigin +) -> tuple[ResearchFailureCause, SavedReferenceFailure | None]: + primary = failure_cause(error, origin) + notice = error if isinstance(error, SavedReferenceFailure) else None + if notice is not None: + primary = primary.model_copy( + update={ + "reason_code": notice.reason_code, + "exception_type": RpcApplicationError.__name__, + } + ) + return primary, notice + + +def research_commit_is_readable( + ledger: LedgerPort, project_id: str, thread_id: str, result: dict[str, JsonValue] +) -> bool: + operation = current_operation.get() + if operation is None or operation.project_id != project_id: + return False + try: + commit = CommitResult.model_validate(result.get("commit")) + except ValidationError: + return False + if ( + commit.receipt.project_id != project_id + or commit.receipt.recorded_at < operation.created_at + or not commit.committed_revision_ids + or not set(commit.committed_revision_ids).issubset(commit.receipt.subject_refs) + ): + return False + work = research_work.get() + if work is not None and ( + work.request_ref.project_id != project_id + or work.request_ref.entity_id != f"request:{thread_id}" + or commit.receipt.receipt_id not in work.produced_receipt_refs + or not set(commit.committed_revision_ids).issubset( + ref.revision_id for ref in work.produced_refs + ) + ): + return False + if commit.receipt not in ledger.read_receipts(project_id): + return False + for identifier in commit.committed_revision_ids: + revision = ledger.read_revision_by_id(project_id, identifier) + if revision is None or revision.project_id != project_id: + return False + snapshot = ledger.read_snapshot(revision.snapshot_id) + if snapshot is None or ( + snapshot.project_id, + snapshot.entity_type, + snapshot.entity_id, + ) != (project_id, revision.entity_type, revision.entity_id): + return False + return True + + +def reference_failure( + ledger: LedgerPort, + project_id: str, + thread_id: str, + result: dict[str, JsonValue], + reason_code: str, +) -> RpcApplicationError: + if re.fullmatch( + r"REFERENCE_[A-Z0-9_]{1,149}", reason_code + ) is not None and research_commit_is_readable(ledger, project_id, thread_id, result): + return SavedReferenceFailure(reason_code, intermediate=research_work.get() is not None) + return RpcApplicationError( + RpcErrorCode.DOMAIN_REJECTED, reason_code, data={"reason_code": reason_code} + ) diff --git a/src/thoth/application/services/research_attempt_runner.py b/src/thoth/application/services/research_attempt_runner.py index bfafd46..1b9ee43 100644 --- a/src/thoth/application/services/research_attempt_runner.py +++ b/src/thoth/application/services/research_attempt_runner.py @@ -10,6 +10,7 @@ from thoth.application.services.model_outcomes import record_model_outcome from thoth.application.services.post_execution_resume import resume_post_execution_learning +from thoth.application.services.reference_failure import reference_failure_cause from thoth.application.services.request_language import apply_request_language from thoth.application.services.research_boundary import RequestBoundary from thoth.application.services.research_termination import ResearchTerminationService @@ -181,7 +182,7 @@ def terminate( ) -> PendingExecution: from thoth.domain.research_failure import FailureOrigin - primary = failure_cause(primary_error, cast(FailureOrigin, origin)) + primary, notice = reference_failure_cause(primary_error, cast(FailureOrigin, origin)) secondary = ( None if secondary_error is None else failure_cause(secondary_error, "HOLD_PUBLICATION") ) @@ -194,7 +195,7 @@ def terminate( ) try: ResearchTerminationService(host.records, host.operations, host.threads).finish_failure( - attempt, request, primary, secondary + attempt, request, primary, secondary, notice=notice ) except ResearchLeaseLost: return PendingExecution({"status": "ATTEMPT_FENCED"}) diff --git a/src/thoth/application/services/research_termination.py b/src/thoth/application/services/research_termination.py index 6cf2369..48c0447 100644 --- a/src/thoth/application/services/research_termination.py +++ b/src/thoth/application/services/research_termination.py @@ -4,6 +4,7 @@ from pydantic import JsonValue +from thoth.application.services.reference_failure import SavedReferenceFailure from thoth.application.services.request_records import RequestRecords from thoth.domain.enums import EntityType, OperationState, ThreadExecutionState from thoth.domain.research_failure import ResearchFailureCause, ResearchFailureRecord @@ -25,6 +26,8 @@ def finish_failure( request: ThreadRequestRevision, primary: ResearchFailureCause, secondary: ResearchFailureCause | None = None, + *, + notice: SavedReferenceFailure | None = None, ) -> ResearchFailureRecord | None: project, op_id = request.project_id, attempt.operation_id key = f"failure:{op_id}:{attempt.attempt_epoch}" @@ -82,10 +85,13 @@ def finish_failure( dict[str, JsonValue], { "code": -32603, - "message": "research execution failed", + "message": "research execution failed" + if notice is None + else notice.message, "data": { "reason_code": primary.reason_code, "failure": failure.model_dump(mode="json"), + **({"research_saved": True} if notice is not None else {}), }, }, ), diff --git a/src/thoth/application/services/sandbox_service.py b/src/thoth/application/services/sandbox_service.py index df61697..b315624 100644 --- a/src/thoth/application/services/sandbox_service.py +++ b/src/thoth/application/services/sandbox_service.py @@ -327,6 +327,8 @@ def accepting_observation( self, spec: SandboxRunSpec, bundle: SandboxExecutionBundle, + *, + defer_rejection: bool = False, ) -> Generator[ArtifactEnvelope, None, None]: try: with self._ledger.transaction(): @@ -385,7 +387,8 @@ def accepting_observation( }, ) except SandboxFailure: - self.reject_observation(bundle, reason="SANDBOX_ADMISSION_BASIS_CHANGED") + if not defer_rejection: + self.reject_observation(bundle, reason="SANDBOX_ADMISSION_BASIS_CHANGED") raise def reject_observation(self, bundle: SandboxExecutionBundle, *, reason: str) -> None: diff --git a/src/thoth/apps/reference_composition.py b/src/thoth/apps/reference_composition.py index 1ff9c54..937309b 100644 --- a/src/thoth/apps/reference_composition.py +++ b/src/thoth/apps/reference_composition.py @@ -40,4 +40,5 @@ def reference_thread_entry( ids=ids, ), ModelReferenceMapper(models, projects, ledger), + ledger, ) diff --git a/src/thoth/apps/resource_scope_composition.py b/src/thoth/apps/resource_scope_composition.py index 025b6b6..5a834a5 100644 --- a/src/thoth/apps/resource_scope_composition.py +++ b/src/thoth/apps/resource_scope_composition.py @@ -31,6 +31,7 @@ from thoth.ports.sandbox import SandboxPort from thoth.ports.store_bundle import StoreBundlePort from thoth.protocol.bus import CommandBus +from thoth.protocol.journal_completion import JournalCompletion from thoth.protocol.operation_owner import OperationOwner from thoth.protocol.registry import MethodRegistry @@ -117,6 +118,7 @@ def scope_bus( queued_admission: Callable[[OperationRecord], dict[str, JsonValue] | None] | None = None, owner: OperationOwner | None = None, journal: JournalPort | None = None, + journal_completion: JournalCompletion | None = None, ) -> CommandBus: handlers = OperationCommandHandlers( operations, @@ -143,6 +145,7 @@ def scope_bus( before_continue=before_continue, queued_admission=queued_admission, owner=owner, + journal_completion=journal_completion, ) diff --git a/src/thoth/apps/runtime_entry.py b/src/thoth/apps/runtime_entry.py index 30c7601..c31cfc9 100644 --- a/src/thoth/apps/runtime_entry.py +++ b/src/thoth/apps/runtime_entry.py @@ -6,7 +6,11 @@ from pathlib import Path from thoth.adapters.operation_host_identity import operation_host_token -from thoth.adapters.storage.workspace_owner import holds_workspace, pin_workspace_owner +from thoth.adapters.storage.workspace_owner import ( + holds_workspace, + pin_workspace_owner, + workspace_owner_key, +) from thoth.application.commands import ThreadCommandHandlers from thoth.application.commands.research_operations import register_research_operations from thoth.application.services import FieldMeasurementService @@ -22,6 +26,7 @@ from thoth.domain.deployment_mode import DeploymentMode from thoth.ports.criterion_contract import CriterionContractStorePort from thoth.ports.evidence_graph import EvidenceGraphStorePort +from thoth.ports.journal import JournalCompletionStorePort from thoth.ports.ledger import ManagedLedgerPort from thoth.ports.model import ModelResolverPort from thoth.ports.model_catalog import ModelCatalogPort @@ -30,6 +35,7 @@ from thoth.ports.store_bundle import StoreBundlePort from thoth.ports.task_profile import TaskProfileCatalogPort from thoth.protocol.bus import CommandBus +from thoth.protocol.journal_completion import JournalCompletion from thoth.protocol.operation_owner import OperationOwner @@ -84,6 +90,7 @@ def assemble_runtime_entry( before_continue = wait_for_hosted_dispatch journal = OperationJournal(stores.events, clock, ids) owner = None + journal_completion = None if mode is DeploymentMode.LOCAL: owner = OperationOwner( worker=research_component.worker, @@ -96,6 +103,16 @@ def assemble_runtime_entry( clock=clock, resource_access=artifact_ledger.scopes, ) + if isinstance(stores.events, JournalCompletionStorePort): + journal_completion = JournalCompletion( + workspace_key=workspace_owner_key(workspace), + pin_workspace=partial(pin_workspace_owner, workspace), + operations=operations, + journal=journal, + store=stores.events, + uow=stores.immediate_uow, + resource_access=artifact_ledger.scopes, + ) bus = scope_bus( registration.registry, ledger, @@ -109,6 +126,7 @@ def assemble_runtime_entry( before_continue=before_continue, queued_admission=research_component.entry.queued_admission, owner=owner, + journal_completion=journal_completion, journal=journal, ) registration.judgment_review.connect(bus, research_component.entry) diff --git a/src/thoth/ports/journal.py b/src/thoth/ports/journal.py index 6c6f05b..f49c358 100644 --- a/src/thoth/ports/journal.py +++ b/src/thoth/ports/journal.py @@ -1,11 +1,13 @@ from __future__ import annotations -from typing import Protocol +from contextlib import AbstractContextManager +from typing import Protocol, runtime_checkable from pydantic import JsonValue from thoth.domain.event import Checkpoint, EventRecord from thoth.domain.operation import OperationRecord +from thoth.ports.runtime import AtomicUnitOfWorkPort class JournalPort(Protocol): @@ -23,3 +25,16 @@ def append( ) -> EventRecord: ... def checkpoint(self, *, operation_id: str, payload: dict[str, JsonValue]) -> Checkpoint: ... + + +class JournalCompletionUnavailable(RuntimeError): + """A repair storage failure; the existing successful result remains authoritative.""" + + +@runtime_checkable +class JournalCompletionStorePort(Protocol): + """Optional complete journal reads and repair-only infrastructure error boundary.""" + + def list_events(self, operation_id: str) -> tuple[EventRecord, ...]: ... + def list_checkpoints(self, operation_id: str) -> tuple[Checkpoint, ...]: ... + def completion_transaction(self, uow: AtomicUnitOfWorkPort) -> AbstractContextManager[None]: ... diff --git a/src/thoth/protocol/bus.py b/src/thoth/protocol/bus.py index 2ebf574..92196c0 100644 --- a/src/thoth/protocol/bus.py +++ b/src/thoth/protocol/bus.py @@ -43,6 +43,11 @@ PendingExecution, current_operation, ) +from thoth.protocol.journal_completion import ( + JournalCompletion, + journal_tail_guard, + operation_result_digest, +) from thoth.protocol.jsonrpc import ( JsonRpcError, JsonRpcRequest, @@ -53,6 +58,7 @@ from thoth.protocol.notifications import notifications_for from thoth.protocol.operation_claim import claim_busy_response, claim_operation from thoth.protocol.operation_owner import OperationOwner, stale_running_error +from thoth.protocol.operation_scope import SYSTEM_PROJECT_METHODS from thoth.protocol.read_queries import READ_QUERY_METHODS from thoth.protocol.registry import CommandHandler, MethodRegistry @@ -84,6 +90,7 @@ def __init__( before_continue: Callable[[str], Awaitable[None]] | None = None, queued_admission: Callable[[OperationRecord], dict[str, JsonValue] | None] | None = None, owner: OperationOwner | None = None, + journal_completion: JournalCompletion | None = None, ) -> None: self._registry = registry self._operations = operations @@ -96,6 +103,7 @@ def __init__( self._before_continue = before_continue self._queued_admission = queued_admission self._owner = owner + self._journal_completion = journal_completion self._operation_task_canceller: Callable[[str], None] | None = None self._failure_diagnostics = ( operations if isinstance(operations, InternalFailureDiagnosticPort) else None @@ -454,30 +462,31 @@ async def _execute(self, ticket: DispatchTicket) -> JsonRpcResponse: ) self._measure(request, operation.project_id, "FAILED") return JsonRpcResponse(id=request.id, error=error) - completed = self._operations.complete( - operation.operation_id, - durable_value, - completed_at=self._clock.now(), - ) - if completed.state is not OperationState.SUCCEEDED: - return self._replay(request.id, completed) - result_digest = domain_digest("OPERATION_RESULT", "1.0.0", canonical_payload(durable_value)) - if self._journal is not None: - self._journal.checkpoint( - operation_id=operation.operation_id, - payload={"state": completed.state.value, "result_digest": result_digest}, - ) - for notification in notifications_for(request.method, durable_value): + with journal_tail_guard(self._journal_completion, operation): + completed = self._operations.complete( + operation.operation_id, + durable_value, + completed_at=self._clock.now(), + ) + if completed.state is not OperationState.SUCCEEDED: + return self._replay(request.id, completed) + result_digest = operation_result_digest(durable_value) + if self._journal is not None: + self._journal.checkpoint( + operation_id=operation.operation_id, + payload={"state": completed.state.value, "result_digest": result_digest}, + ) + for notification in notifications_for(request.method, durable_value): + self._append_event( + operation, + notification, + {"method": request.method, "result_digest": result_digest}, + ) self._append_event( operation, - notification, - {"method": request.method, "result_digest": result_digest}, + "operation.succeeded", + {"result_digest": result_digest}, ) - self._append_event( - operation, - "operation.succeeded", - {"result_digest": result_digest}, - ) self._measure(request, operation.project_id, "SUCCEEDED") response = self._result_payload(completed) if isinstance(value, EphemeralCommandResult): @@ -499,18 +508,7 @@ def _measure(self, request: JsonRpcRequest, project_id: str, outcome: str) -> No return def _project_scope(self, request: JsonRpcRequest) -> str: - if request.method in { - "workspace/setup/read", - "workspace/setup/update", - "workspace/ready", - "model/credential/list", - "model/credential/register", - "model/credential/login/status", - "model/credential/login/cancel", - "model/credential/login/complete", - "model/catalog/refresh", - "model/tooling/install", - }: + if request.method in SYSTEM_PROJECT_METHODS: return "system:workspace" value = request.params.input.get("project_id") if not isinstance(value, str) or not value: @@ -607,6 +605,8 @@ def _fail_abandoned_running( def _replay(self, request_id: str | int, operation: OperationRecord) -> JsonRpcResponse: if self._resource_access is not None: self._resource_access.require_operation(operation) + if self._journal_completion is not None: + operation = self._journal_completion.repair(operation) if operation.state == OperationState.SUCCEEDED and operation.result is not None: return JsonRpcResponse(id=request_id, result=self._result_payload(operation)) if operation.state == OperationState.FAILED and operation.error is not None: diff --git a/src/thoth/protocol/journal_completion.py b/src/thoth/protocol/journal_completion.py new file mode 100644 index 0000000..0c99261 --- /dev/null +++ b/src/thoth/protocol/journal_completion.py @@ -0,0 +1,242 @@ +"""Supplement a measured success tail without repeating effects or changing its result.""" + +from collections import Counter +from collections.abc import Callable, Generator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from typing import Literal + +from pydantic import JsonValue + +from thoth.domain.canonical import canonical_payload, domain_digest +from thoth.domain.enums import OperationState +from thoth.domain.event import EventRecord +from thoth.domain.operation import OperationRecord +from thoth.domain.resource_scope import ResourceScopeError, resource_use_verification +from thoth.ports.journal import ( + JournalCompletionStorePort, + JournalCompletionUnavailable, + JournalPort, +) +from thoth.ports.operation import OperationStorePort +from thoth.ports.resource_scope import ResourceAccessPort +from thoth.ports.runtime import AtomicUnitOfWorkPort +from thoth.protocol.journal_completion_gate import journal_gate +from thoth.protocol.journal_completion_methods import SUPPLEMENTABLE_METHODS, supported_result +from thoth.protocol.notifications import notifications_for + + +def operation_result_digest(value: dict[str, JsonValue]) -> str: + return domain_digest("OPERATION_RESULT", "1.0.0", canonical_payload(value)) + + +@dataclass(frozen=True) +class CompletionOutcome: + operation: OperationRecord + status: Literal["NOT_ELIGIBLE", "COMPLETE", "SUPPLEMENTED", "DEFERRED"] + added: int = 0 + reason: str | None = None + + +@dataclass(frozen=True) +class _Missing: + checkpoint: bool + notifications: tuple[str, ...] + succeeded: bool + + @property + def count(self) -> int: + return int(self.checkpoint) + len(self.notifications) + int(self.succeeded) + + +class _Deferred(Exception): + pass + + +def _payload_matches( + actual: dict[str, object], base: dict[str, JsonValue], completed_at: str +) -> bool: + if actual == base: + return True + return ( + actual == {**base, "supplemented": True, "supplemented_for": completed_at} + and actual.get("supplemented") is True + ) + + +def _chain_valid(events: tuple[EventRecord, ...], operation: OperationRecord) -> bool: + if len({event.event_id for event in events}) != len(events): + return False + previous = None + for event in events: + if ( + event.operation_id != operation.operation_id + or event.project_id != operation.project_id + or event.schema_version != "1.0.0" + or event.previous_event_digest != previous + or event.event_digest + != domain_digest( + "OPERATION_EVENT", + "1.0.0", + canonical_payload(event.model_dump(mode="python", exclude={"event_digest"})), + ) + ): + return False + previous = event.event_digest + return True + + +class JournalCompletion: + def __init__( + self, + *, + workspace_key: str, + pin_workspace: Callable[[], AbstractContextManager[bool]], + operations: OperationStorePort, + journal: JournalPort, + store: JournalCompletionStorePort, + uow: AtomicUnitOfWorkPort, + resource_access: ResourceAccessPort, + ) -> None: + self.workspace_key = workspace_key + self._pin = pin_workspace + self._operations = operations + self._journal = journal + self._store = store + self._uow = uow + self._access = resource_access + + def repair(self, operation: OperationRecord) -> OperationRecord: + return self.attempt(operation).operation + + def attempt(self, operation: OperationRecord) -> CompletionOutcome: + if ( + operation.state is not OperationState.SUCCEEDED + or operation.method not in SUPPLEMENTABLE_METHODS + ): + return CompletionOutcome(operation, "NOT_ELIGIBLE") + if ( + operation.result is None + or operation.completed_at is None + or operation.error is not None + or operation.schema_version != "1.0.0" + or not supported_result(operation.method, operation.result) + ): + return CompletionOutcome(operation, "DEFERRED", reason="UNSUPPORTED_SAVED_FORM") + with journal_gate(self.workspace_key, operation.operation_id), self._pin() as held: + if not held: + return CompletionOutcome(operation, "DEFERRED", reason="WORKSPACE_NOT_OWNED") + try: + with ( + resource_use_verification(operation.project_id), + self._store.completion_transaction(self._uow), + ): + current = self._operations.read(operation.operation_id) + if current is None: + raise ResourceScopeError("OPERATION_RESOURCE_BINDING_INVALID") + self._access.require_operation(current) + if current.model_dump(mode="json") != operation.model_dump(mode="json"): + raise ResourceScopeError("OPERATION_RESOURCE_BINDING_INVALID") + missing = self._inspect(current) + if not missing.count: + return CompletionOutcome(operation, "COMPLETE") + self._write(current, missing) + if self._inspect(current).count: + raise _Deferred("INCOMPLETE_AFTER_APPEND") + return CompletionOutcome(operation, "SUPPLEMENTED", added=missing.count) + except _Deferred as exc: + return CompletionOutcome(operation, "DEFERRED", reason=str(exc)) + except JournalCompletionUnavailable: + return CompletionOutcome(operation, "DEFERRED", reason="STORAGE_UNAVAILABLE") + + def _inspect(self, operation: OperationRecord) -> _Missing: + assert operation.result is not None and operation.completed_at is not None + digest = operation_result_digest(operation.result) + completed_at = operation.completed_at.isoformat() + expected = notifications_for(operation.method, operation.result) + events = self._store.list_events(operation.operation_id) + points = self._store.list_checkpoints(operation.operation_id) + if not events: + raise _Deferred("MISSING_STARTED") + if not _chain_valid(events, operation): + raise _Deferred("UNSAFE_EVENT_CHAIN_OR_ORDER") + if events[0].event_type != "operation.started": + raise _Deferred("MISSING_STARTED") + counts = Counter(event.event_type for event in events) + if any(count != 1 for count in counts.values()): + raise _Deferred("DUPLICATE_EVENT") + for event in events: + if event.event_type == "operation.started": + if event.payload != {"method": operation.method}: + raise _Deferred("UNSUPPORTED_STARTED_PAYLOAD") + continue + if event.event_type == "operation.succeeded": + base: dict[str, JsonValue] = {"result_digest": digest} + elif event.event_type in expected: + base = {"method": operation.method, "result_digest": digest} + else: + raise _Deferred("OTHER_PUBLISHER_EVENT") + if not _payload_matches(event.payload, base, completed_at): + raise _Deferred("EVENT_PAYLOAD_MISMATCH") + if len(points) > 1: + raise _Deferred("MULTIPLE_CHECKPOINTS") + if points: + point = points[0] + if ( + point.operation_id != operation.operation_id + or not _payload_matches( + point.payload, {"state": "SUCCEEDED", "result_digest": digest}, completed_at + ) + or point.checkpoint_digest + != domain_digest( + "OPERATION_CHECKPOINT", + "1.0.0", + canonical_payload( + point.model_dump(mode="python", exclude={"checkpoint_digest"}) + ), + ) + ): + raise _Deferred("CHECKPOINT_PAYLOAD_MISMATCH") + return _Missing( + not points, + tuple(name for name in expected if name not in counts), + "operation.succeeded" not in counts, + ) + + def _write(self, operation: OperationRecord, missing: _Missing) -> None: + assert operation.result is not None and operation.completed_at is not None + digest = operation_result_digest(operation.result) + mark: dict[str, JsonValue] = { + "supplemented": True, + "supplemented_for": operation.completed_at.isoformat(), + } + if missing.checkpoint: + self._journal.checkpoint( + operation_id=operation.operation_id, + payload={"state": "SUCCEEDED", "result_digest": digest, **mark}, + ) + for notification in missing.notifications: + self._journal.append( + project_id=operation.project_id, + operation_id=operation.operation_id, + event_type=notification, + payload={"method": operation.method, "result_digest": digest, **mark}, + ) + if missing.succeeded: + self._journal.append( + project_id=operation.project_id, + operation_id=operation.operation_id, + event_type="operation.succeeded", + payload={"result_digest": digest, **mark}, + ) + + +@contextmanager +def journal_tail_guard( + completion: JournalCompletion | None, operation: OperationRecord +) -> Generator[None]: + if completion is None or operation.method not in SUPPLEMENTABLE_METHODS: + yield + else: + with journal_gate(completion.workspace_key, operation.operation_id): + yield diff --git a/src/thoth/protocol/journal_completion_gate.py b/src/thoth/protocol/journal_completion_gate.py new file mode 100644 index 0000000..fcc89df --- /dev/null +++ b/src/thoth/protocol/journal_completion_gate.py @@ -0,0 +1,34 @@ +"""Reentrant success-tail gates shared by LOCAL runtimes in one process.""" + +from _thread import RLock +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from threading import Lock + + +@dataclass +class _Entry: + lock: RLock = field(default_factory=RLock) + users: int = 0 + + +_entries: dict[tuple[str, str], _Entry] = {} +_registry_lock = Lock() + + +@contextmanager +def journal_gate(workspace_key: str, operation_id: str) -> Generator[None]: + """Count holders and waiters, then discard the entry after the last exit.""" + key = (workspace_key, operation_id) + with _registry_lock: + entry = _entries.setdefault(key, _Entry()) + entry.users += 1 + try: + with entry.lock: + yield + finally: + with _registry_lock: + entry.users -= 1 + if entry.users == 0: + del _entries[key] diff --git a/src/thoth/protocol/journal_completion_methods.py b/src/thoth/protocol/journal_completion_methods.py new file mode 100644 index 0000000..5e7a59f --- /dev/null +++ b/src/thoth/protocol/journal_completion_methods.py @@ -0,0 +1,213 @@ +"""V-J measured durable result forms; unknown methods and forms are not repaired.""" + +from pydantic import JsonValue + +_FORM_1 = frozenset( + { + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_2 = frozenset( + { + "cutoff_at", + "description", + "impact", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_3 = frozenset( + { + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "reference", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_4 = frozenset( + { + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "role", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams", + } +) + +_FORM_5 = frozenset( + { + "baseline_candidate", + "impact", + "protected_authority_package", + } +) + +_FORM_6 = frozenset( + { + "branch", + "content_mutated", + } +) + +_FORM_7 = frozenset( + { + "change_set", + "conflict_state", + "missing_dependencies", + } +) + +_FORM_8 = frozenset( + { + "change_set", + "checks", + "impact_propagation_plan", + "state", + "validation_bundle_digest", + } +) + +_FORM_9 = frozenset( + { + "change_set", + "impact_propagation_plan", + "state", + "validation_bundle_digest", + } +) + +_FORM_10 = frozenset( + { + "conflict", + "head_mutated", + "merge_proposal", + "receipt", + "semantic_merge", + } +) + +_FORM_11 = frozenset( + { + "candidate_revision", + "change_set", + "head_mutated", + "merge_proposal", + } +) + +_FORM_12 = frozenset( + { + "impact_preview", + "proposal", + "staged_snapshot", + "validation_state", + } +) + +_FORM_13 = frozenset( + { + "recompute_task", + } +) + +_FORM_14 = frozenset( + { + "change_set", + "external_effect_rollback_claimed", + "impact_plan", + "restore_proposal", + } +) + +SUPPORTED_RESULT_FORMS: dict[str, tuple[frozenset[str], ...]] = { + "project/activate": (_FORM_1,), + "project/archive": (_FORM_1,), + "project/create": (_FORM_1,), + "project/cutoff/update": (_FORM_2,), + "project/metadata/update": (_FORM_1,), + "project/overlay/update": (_FORM_1,), + "project/policy/update": (_FORM_1,), + "project/reference/import": (_FORM_3,), + "project/restore": (_FORM_1,), + "project/role/assign": (_FORM_4,), + "project/role/revoke": (_FORM_4,), + "revision/baseline/prepare": (_FORM_5,), + "revision/branch/create": (_FORM_6,), + "revision/changeSet/create": (_FORM_7,), + "revision/changeSet/validate": ( + _FORM_8, + _FORM_9, + ), + "revision/merge/propose": (_FORM_10,), + "revision/merge/resolve": (_FORM_11,), + "revision/propose": (_FORM_12,), + "revision/recompute/request": (_FORM_13,), + "revision/restore/propose": (_FORM_14,), +} +SUPPLEMENTABLE_METHODS = frozenset(SUPPORTED_RESULT_FORMS) + + +def supported_result(method: str, result: dict[str, JsonValue]) -> bool: + return ( + frozenset(result) in SUPPORTED_RESULT_FORMS.get(method, ()) + and result.get("schema_version", "1.0.0") == "1.0.0" + ) diff --git a/src/thoth/protocol/operation_scope.py b/src/thoth/protocol/operation_scope.py new file mode 100644 index 0000000..3e3105d --- /dev/null +++ b/src/thoth/protocol/operation_scope.py @@ -0,0 +1,16 @@ +"""Existing system RPCs that do not require input.project_id.""" + +SYSTEM_PROJECT_METHODS = frozenset( + { + "workspace/setup/read", + "workspace/setup/update", + "workspace/ready", + "model/credential/list", + "model/credential/register", + "model/credential/login/status", + "model/credential/login/cancel", + "model/credential/login/complete", + "model/catalog/refresh", + "model/tooling/install", + } +) diff --git a/tests/fixtures/operation_result_digest_vectors.json b/tests/fixtures/operation_result_digest_vectors.json new file mode 100644 index 0000000..cc2d59b --- /dev/null +++ b/tests/fixtures/operation_result_digest_vectors.json @@ -0,0 +1,44 @@ +{ + "source": "CP56 OPERATION_RESULT/1.0.0 formula; independent of J helper", + "vectors": [ + { + "value": {}, + "digest": "bca32c7f85e8328c9805776a42733883e04f0f358c2b97ac00a9538e2b4f7e9f" + }, + { + "value": { + "revision": 1, + "name": "Changed exactly once" + }, + "digest": "f5659a222b58a5109567afbe95c662e7ec9fb12c2002ea4ca9be9a1c85376785" + }, + { + "value": { + "z": "한국어 · English", + "a": [ + null, + false, + 7 + ] + }, + "digest": "d312b684a88236c57206e1e3a43a2ffd69c33845935a464fc71498032bbb07c6" + }, + { + "value": { + "nested": { + "b": "2.5", + "a": "value" + }, + "empty": [] + }, + "digest": "1a59a9351dc1d90ec328496bb7354172bb4ad58a8ee46382cf3df59fa6ea4c82" + }, + { + "value": { + "state": "SUCCEEDED", + "unchanged": true + }, + "digest": "a57f47caee392c540518a8c1705345305f5e60a74d23bb6fbdaccd73a376386f" + } + ] +} diff --git a/tests/fixtures/r2_closed_loop_golden.json b/tests/fixtures/r2_closed_loop_golden.json index fb73be1..abe4ad3 100644 --- a/tests/fixtures/r2_closed_loop_golden.json +++ b/tests/fixtures/r2_closed_loop_golden.json @@ -4122,6 +4122,10 @@ "PreparedResearchExecution", "SandboxExecutionBundle" ], + [ + "L", + "transaction" + ], [ "X", "reject_observation", @@ -7475,11 +7479,16 @@ "_measurement_assessments", "tuple:d01bf921a2fd20c7" ], + [ + "L", + "transaction" + ], [ "X", "accepting_observation", "SandboxRunSpec:92bdabb537c574c0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -13069,7 +13078,8 @@ "X", "accepting_observation", "SandboxRunSpec:92bdabb537c574c0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -13576,6 +13586,10 @@ "project:a04:allowed", "d0a6cfbed30739d9a4438f597479803ad7436faf747e04fb2135ecd2662cb0eb" ], + [ + "L", + "transaction" + ], [ "X", "reject_observation", @@ -17276,7 +17290,8 @@ "X", "accepting_observation", "SandboxRunSpec:92bdabb537c574c0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -23143,7 +23158,8 @@ "X", "accepting_observation", "SandboxRunSpec:39352fb13a8e9978", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -27229,7 +27245,8 @@ "X", "accepting_observation", "SandboxRunSpec:60e264ea04d64b51", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -30951,7 +30968,8 @@ "X", "accepting_observation", "SandboxRunSpec:647dffad191532ca", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -32223,7 +32241,8 @@ "X", "accepting_observation", "SandboxRunSpec:ae549f5c7a2d01bf", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -32525,7 +32544,8 @@ "X", "accepting_observation", "SandboxRunSpec:87ac021e9c054bf0", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -33785,7 +33805,8 @@ "X", "accepting_observation", "SandboxRunSpec:8481b5ba1239c926", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -35045,7 +35066,8 @@ "X", "accepting_observation", "SandboxRunSpec:b9f2d962054617b7", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -36518,7 +36540,8 @@ "X", "accepting_observation", "SandboxRunSpec:0848943b2e1ca6a2", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -36958,7 +36981,8 @@ "X", "accepting_observation", "SandboxRunSpec:59a09a8fd8a8c946", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", @@ -39468,6 +39492,10 @@ "read_heads", "project:a04:allowed" ], + [ + "L", + "transaction" + ], [ "S", "enter", @@ -40859,7 +40887,8 @@ "X", "accepting_observation", "SandboxRunSpec:5e5ec03e176405ed", - "SandboxExecutionBundle" + "SandboxExecutionBundle", + "defer_rejection=True" ], [ "L", diff --git a/tests/fixtures/reference_service_golden.json b/tests/fixtures/reference_service_golden.json index 3ddc553..be9703e 100644 --- a/tests/fixtures/reference_service_golden.json +++ b/tests/fixtures/reference_service_golden.json @@ -7,9 +7,10 @@ "error": { "code": -32030, "data": { - "reason_code": "REFERENCE_ANSWER_REVISION_CONFLICT" + "reason_code": "REFERENCE_ANSWER_REVISION_CONFLICT", + "research_saved": true }, - "message": "REFERENCE_ANSWER_REVISION_CONFLICT" + "message": "\uc5f0\uad6c \uacb0\uacfc\ub294 \uc800\uc7a5\ub410\uc2b5\ub2c8\ub2e4. \uae30\uc900(reference) \ub2e8\uacc4\ub9cc \uc2e4\ud328\ud588\uc2b5\ub2c8\ub2e4 (\uc0ac\uc720: REFERENCE_ANSWER_REVISION_CONFLICT). \uacb0\uacfc\ub294 \uae30\ub85d\uc5d0\uc11c \ubcfc \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uac19\uc740 \uc9c8\ubb38\uc744 \uc0c8\ub85c \ubcf4\ub0b4\uba74 \uc5f0\uad6c\uac00 \ub2e4\uc2dc \uc2e4\ud589\ub420 \uc218 \uc788\uc73c\ub2c8 \uba3c\uc800 \uc800\uc7a5\ub41c \uacb0\uacfc\ub97c \ud655\uc778\ud558\uc138\uc694. / The research result was saved. Only the reference step failed (reason: REFERENCE_ANSWER_REVISION_CONFLICT). You can read the result in the history. Sending the same question again as a new request may run the research again, so check the saved result first." } } ] diff --git a/tests/fixtures/supplementable_methods.json b/tests/fixtures/supplementable_methods.json new file mode 100644 index 0000000..95fcfb8 --- /dev/null +++ b/tests/fixtures/supplementable_methods.json @@ -0,0 +1,341 @@ +{ + "schema_version": "1.0.0", + "source_head": "70e6739f08c14a3f54f0ec43adfd46b487fcf4b1", + "methods": [ + "project/activate", + "project/archive", + "project/create", + "project/cutoff/update", + "project/metadata/update", + "project/overlay/update", + "project/policy/update", + "project/reference/import", + "project/restore", + "project/role/assign", + "project/role/revoke", + "revision/baseline/prepare", + "revision/branch/create", + "revision/changeSet/create", + "revision/changeSet/validate", + "revision/merge/propose", + "revision/merge/resolve", + "revision/propose", + "revision/recompute/request", + "revision/restore/propose" + ], + "result_key_shapes": { + "project/activate": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/archive": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/create": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/cutoff/update": [ + [ + "cutoff_at", + "description", + "impact", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/metadata/update": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/overlay/update": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/policy/update": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/reference/import": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "reference", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/restore": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/role/assign": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "role", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "project/role/revoke": [ + [ + "cutoff_at", + "description", + "lifecycle", + "name", + "overlay", + "policy", + "policy_binding_ref", + "project_id", + "public_web_execution", + "references", + "revision", + "role", + "roles", + "schema_version", + "source_binding_ids", + "source_bindings", + "working_heads", + "workstreams" + ] + ], + "revision/baseline/prepare": [ + [ + "baseline_candidate", + "impact", + "protected_authority_package" + ] + ], + "revision/branch/create": [ + [ + "branch", + "content_mutated" + ] + ], + "revision/changeSet/create": [ + [ + "change_set", + "conflict_state", + "missing_dependencies" + ] + ], + "revision/changeSet/validate": [ + [ + "change_set", + "checks", + "impact_propagation_plan", + "state", + "validation_bundle_digest" + ], + [ + "change_set", + "impact_propagation_plan", + "state", + "validation_bundle_digest" + ] + ], + "revision/merge/propose": [ + [ + "conflict", + "head_mutated", + "merge_proposal", + "receipt", + "semantic_merge" + ] + ], + "revision/merge/resolve": [ + [ + "candidate_revision", + "change_set", + "head_mutated", + "merge_proposal" + ] + ], + "revision/propose": [ + [ + "impact_preview", + "proposal", + "staged_snapshot", + "validation_state" + ] + ], + "revision/recompute/request": [ + [ + "recompute_task" + ] + ], + "revision/restore/propose": [ + [ + "change_set", + "external_effect_rollback_claimed", + "impact_plan", + "restore_proposal" + ] + ] + }, + "evidence": { + "path": "outputs/refactor-vj-20261019/method-evidence.json", + "sha256": "28f24982437dbab8f8fa637e066e10fc730ef3da1af9822e51df4ab69c1374eb" + }, + "scope": "LOCAL; only measured saved result shapes" +} diff --git a/tests/integration/journal_completion_helpers.py b/tests/integration/journal_completion_helpers.py new file mode 100644 index 0000000..576b8b7 --- /dev/null +++ b/tests/integration/journal_completion_helpers.py @@ -0,0 +1,81 @@ +"""Real LOCAL setup with a committed result and a deliberately absent success tail.""" + +from dataclasses import dataclass +from pathlib import Path + +import pytest +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.apps.runtime import AppRuntime +from thoth.domain.deployment_mode import DeploymentMode +from thoth.domain.event import Checkpoint +from thoth.domain.operation import OperationRecord +from thoth.protocol.bus import DispatchTicket +from thoth.protocol.jsonrpc import JsonRpcRequest + + +@dataclass +class Gap: + runtime: AppRuntime + command: JsonRpcRequest + operation: OperationRecord + model: NoModels + + +class TailInterrupted(RuntimeError): + pass + + +async def prepare_gap( + workspace: Path, + monkeypatch: pytest.MonkeyPatch, + *, + mode: DeploymentMode = DeploymentMode.LOCAL, +) -> Gap: + model = NoModels() + runtime = create_runtime(workspace, model_resolver=model, deployment_mode=mode) + try: + value( + await runtime.bus.dispatch( + request( + "project/create", + "create", + { + "project_id": "project:j", + "name": "Original", + "cutoff_at": "2026-10-01T00:00:00Z", + }, + ) + ) + ) + command = request( + "project/metadata/update", + "gap", + { + "project_id": "project:j", + "expected_revision": 0, + "name": "Saved once", + }, + ) + ticket = runtime.bus.claim(command) + assert isinstance(ticket, DispatchTicket) + original = SqliteEventStore.insert_checkpoint + + def fail(store: SqliteEventStore, point: Checkpoint) -> None: + if point.operation_id == ticket.operation.operation_id: + raise TailInterrupted("checkpoint before insert") + original(store, point) + + with monkeypatch.context() as patch: + patch.setattr(SqliteEventStore, "insert_checkpoint", fail) + with pytest.raises(TailInterrupted): + await runtime.bus.execute(ticket) + stored = runtime.bus.read_operation(ticket.operation.operation_id) + assert stored is not None and stored.result is not None + return Gap(runtime, command, stored, model) + except BaseException: + runtime.close() + raise diff --git a/tests/integration/p_r2_failure_helpers.py b/tests/integration/p_r2_failure_helpers.py new file mode 100644 index 0000000..3452256 --- /dev/null +++ b/tests/integration/p_r2_failure_helpers.py @@ -0,0 +1,83 @@ +"""Normal-entry R2 fault instrumentation; snapshots use independent connections.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.test_hypothesis_prediction_outcome_cycle import request, value + +from thoth.adapters.storage import transaction as transaction_module +from thoth.application.services.r2_closed_loop import R2ClosedLoopCoordinator +from thoth.apps.runtime import AppRuntime +from thoth.domain.sandbox import SandboxRunSpec + + +def ambient() -> bool: + return vars(transaction_module)["_AMBIENT"].get() is not None + + +def save(name: str, evidence: dict[str, Any]) -> None: + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/refactor-p-20261022" + / os.environ.get("THOTH_P_RUN_LABEL", "p2-direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{name}.json").write_text( + json.dumps(evidence, indent=2, default=str) + "\n", encoding="utf-8" + ) + + +def capture_exit(patch: pytest.MonkeyPatch, runtime: AppRuntime, evidence: dict[str, Any]) -> None: + original = R2ClosedLoopCoordinator.execute_if_required + + async def execute(coordinator: R2ClosedLoopCoordinator, **kwargs: Any) -> Any: + try: + return await original(coordinator, **kwargs) + finally: + # This runs before the outer handler publishes its terminal operation/error. + evidence["after_phase"] = snapshot(runtime.ledger.engine) + evidence["ambient_after_phase"] = ambient() + + patch.setattr(R2ClosedLoopCoordinator, "execute_if_required", execute) + + +async def revise_hypothesis(runtime: AppRuntime, project: str, spec: SandboxRunSpec) -> None: + action = value( + await runtime.bus.dispatch( + request( + "action/read", "p2-action", {"project_id": project, "action_id": spec.action_id} + ) + ) + )["action"] + hypothesis_id = action["hypothesis_refs"][0] + hypothesis = value( + await runtime.bus.dispatch( + request( + "hypothesis/read", + "p2-hypothesis", + {"project_id": project, "hypothesis_id": hypothesis_id}, + ) + ) + )["hypothesis"] + value( + await runtime.bus.dispatch( + request( + "hypothesis/revise", + "p2-concurrent-edit", + { + "project_id": project, + "hypothesis_id": hypothesis_id, + "expected_revision_digest": hypothesis["revision_digest"], + "patch": {"statement": "Operator revised the measurement hypothesis"}, + "evidence_refs": hypothesis["evidence_refs"], + "reason": "new operator evidence", + }, + ) + ) + ) diff --git a/tests/integration/test_journal_completion_concurrency.py b/tests/integration/test_journal_completion_concurrency.py new file mode 100644 index 0000000..ee70add --- /dev/null +++ b/tests/integration/test_journal_completion_concurrency.py @@ -0,0 +1,127 @@ +"""Real SQLite callers coordinate normal tails and repeated repairs across runtimes.""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Event +from time import monotonic + +import pytest +from tests.integration.journal_completion_helpers import TailInterrupted, prepare_gap +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.workspace_owner import workspace_owner_key +from thoth.domain.event import Checkpoint +from thoth.protocol import journal_completion_gate as gates +from thoth.protocol.bus import DispatchTicket + + +async def test_two_runtime_replays_add_only_one_tail( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + second = create_runtime(tmp_path, model_resolver=gap.model) + try: + with ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(asyncio.run, gap.runtime.bus.dispatch(gap.command)) + b = pool.submit(asyncio.run, second.bus.dispatch(gap.command)) + left, right = a.result(timeout=10), b.result(timeout=10) + assert left == right and value(left) == gap.operation.result + events = SqliteEventStore(gap.runtime.ledger.engine) + assert [e.event_type for e in events.list_events(gap.operation.operation_id)] == [ + "operation.started", + "project/metadata/updated", + "operation.succeeded", + ] + assert len(events.list_checkpoints(gap.operation.operation_id)) == 1 + assert gap.model.calls == 0 + assert (workspace_owner_key(tmp_path), gap.operation.operation_id) not in vars(gates)[ + "_entries" + ] + finally: + second.close() + gap.runtime.close() + + +@pytest.mark.parametrize("normal_fails", [False, True]) +async def test_replay_waits_for_original_tail_before_deciding_what_is_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, normal_fails: bool +) -> None: + model = NoModels() + first = create_runtime(tmp_path, model_resolver=model) + second = create_runtime(tmp_path, model_resolver=model) + waiting, release = Event(), Event() + try: + value( + await first.bus.dispatch( + request( + "project/create", + "create", + { + "project_id": "project:j", + "name": "Original", + "cutoff_at": "2026-10-01T00:00:00Z", + }, + ) + ) + ) + command = request( + "project/metadata/update", + "race", + { + "project_id": "project:j", + "expected_revision": 0, + "name": "Saved once", + }, + ) + ticket = first.bus.claim(command) + assert isinstance(ticket, DispatchTicket) + identifier = ticket.operation.operation_id + original = SqliteEventStore.insert_checkpoint + + def paused(self: SqliteEventStore, point: Checkpoint) -> None: + if point.operation_id == identifier and self._engine is first.ledger.engine: # pyright: ignore[reportPrivateUsage] + waiting.set() + assert release.wait(timeout=10) + if normal_fails: + raise TailInterrupted("original tail interrupted") + original(self, point) + + monkeypatch.setattr(SqliteEventStore, "insert_checkpoint", paused) + with ThreadPoolExecutor(max_workers=2) as pool: + writer = pool.submit(asyncio.run, first.bus.execute(ticket)) + try: + assert waiting.wait(timeout=10) + replay = pool.submit(asyncio.run, second.bus.dispatch(command)) + key = (workspace_owner_key(tmp_path), identifier) + deadline = monotonic() + 10 + while vars(gates)["_entries"][key].users < 2 and monotonic() < deadline: + Event().wait(0.005) + assert vars(gates)["_entries"][key].users == 2 + assert not replay.done() + release.set() + if normal_fails: + with pytest.raises(TailInterrupted): + writer.result(timeout=10) + else: + assert writer.result(timeout=10) == replay.result(timeout=10) + assert value(replay.result(timeout=10))["revision"] == 1 + finally: + release.set() + store = SqliteEventStore(second.ledger.engine) + events = store.list_events(identifier) + assert [e.event_type for e in events] == [ + "operation.started", + "project/metadata/updated", + "operation.succeeded", + ] + assert len(store.list_checkpoints(identifier)) == 1 + assert all((e.payload.get("supplemented") is True) is normal_fails for e in events[1:]) + assert key not in vars(gates)["_entries"] and model.calls == 0 + finally: + release.set() + second.close() + first.close() diff --git a/tests/integration/test_journal_completion_faults.py b/tests/integration/test_journal_completion_faults.py new file mode 100644 index 0000000..9652753 --- /dev/null +++ b/tests/integration/test_journal_completion_faults.py @@ -0,0 +1,143 @@ +"""Real repair transactions defer infrastructure failures but preserve authority failures.""" + +from contextlib import nullcontext +from datetime import datetime, timedelta +from pathlib import Path +from typing import cast + +import pytest +from sqlalchemy.exc import OperationalError +from tests.atomicity.harness import snapshot +from tests.integration.journal_completion_helpers import prepare_gap +from tests.integration.storage_coverage_helpers import value + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.application.services.operation_journal import OperationJournal +from thoth.application.services.resource_scope_service import ResourceScopeService +from thoth.domain.deployment_mode import DeploymentMode +from thoth.domain.event import EventRecord +from thoth.domain.operation import OperationRecord +from thoth.domain.resource_scope import ResourceScopeError +from thoth.protocol.journal_completion import JournalCompletion + + +async def test_operational_write_failure_rolls_back_and_preserves_success( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + original = SqliteEventStore.insert_event + try: + before = snapshot(gap.runtime.ledger.engine) + + def failed(self: SqliteEventStore, event: EventRecord) -> None: + original(self, event) + if event.operation_id == gap.operation.operation_id: + raise OperationalError("synthetic journal I/O", {}, OSError("synthetic")) + + with monkeypatch.context() as patch: + patch.setattr(SqliteEventStore, "insert_event", failed) + response = await gap.runtime.bus.dispatch(gap.command) + assert value(response) == gap.operation.result + assert snapshot(gap.runtime.ledger.engine) == before + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + events = SqliteEventStore(gap.runtime.ledger.engine) + assert len(events.list_events(gap.operation.operation_id)) == 3 + assert len(events.list_checkpoints(gap.operation.operation_id)) == 1 + assert gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_replay_authority_denial_happens_before_any_journal_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + before = snapshot(gap.runtime.ledger.engine) + + def denied(self: ResourceScopeService, operation: OperationRecord) -> None: + assert operation.operation_id == gap.operation.operation_id + raise ResourceScopeError("RESOURCE_SCOPE_READ_DENIED") + + monkeypatch.setattr(ResourceScopeService, "require_operation", denied) + response = await gap.runtime.bus.dispatch(gap.command) + assert response.error is not None + assert response.error.data["reason_code"] == "RESOURCE_SCOPE_READ_DENIED" + assert snapshot(gap.runtime.ledger.engine) == before and gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_corrupt_result_binding_is_not_hidden_by_repair( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + with gap.runtime.ledger.engine.begin() as connection: + connection.exec_driver_sql( + "UPDATE operations SET result_json=? WHERE operation_id=?", + ('{"tampered":true}', gap.operation.operation_id), + ) + before = snapshot(gap.runtime.ledger.engine) + response = await gap.runtime.bus.dispatch(gap.command) + assert response.error is not None + assert response.error.data["reason_code"] == "OPERATION_RESOURCE_BINDING_INVALID" + assert snapshot(gap.runtime.ledger.engine) == before and gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_missing_workspace_pin_defers_without_acquiring_a_new_lock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + completion = cast(JournalCompletion, vars(gap.runtime.bus)["_journal_completion"]) + monkeypatch.setattr(completion, "_pin", lambda: nullcontext(False)) + before = snapshot(gap.runtime.ledger.engine) + assert completion.attempt(gap.operation).reason == "WORKSPACE_NOT_OWNED" + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + assert snapshot(gap.runtime.ledger.engine) == before + finally: + gap.runtime.close() + + +async def test_hosted_replay_does_not_supplement( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch, mode=DeploymentMode.HOSTED_REVIEW) + try: + assert vars(gap.runtime.bus)["_journal_completion"] is None + before = snapshot(gap.runtime.ledger.engine) + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + assert snapshot(gap.runtime.ledger.engine) == before and gap.model.calls == 0 + finally: + gap.runtime.close() + + +async def test_sqlite_repair_rolls_back_clock_order_conflict( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + gap = await prepare_gap(tmp_path, monkeypatch) + try: + store = SqliteEventStore(gap.runtime.ledger.engine) + old_time = store.list_events(gap.operation.operation_id)[0].created_at - timedelta( + seconds=1 + ) + + class BackwardClock: + def now(self) -> datetime: + return old_time + + completion = cast(JournalCompletion, vars(gap.runtime.bus)["_journal_completion"]) + journal = cast(OperationJournal, vars(gap.runtime.bus)["_journal"]) + before = snapshot(gap.runtime.ledger.engine) + with monkeypatch.context() as patch: + patch.setattr(journal, "_clock", BackwardClock()) + outcome = completion.attempt(gap.operation) + assert outcome.status == "DEFERRED" and outcome.reason == "UNSAFE_EVENT_CHAIN_OR_ORDER" + assert snapshot(gap.runtime.ledger.engine) == before + assert value(await gap.runtime.bus.dispatch(gap.command)) == gap.operation.result + assert len(store.list_events(gap.operation.operation_id)) == 3 + finally: + gap.runtime.close() diff --git a/tests/integration/test_journal_completion_process.py b/tests/integration/test_journal_completion_process.py new file mode 100644 index 0000000..87a99eb --- /dev/null +++ b/tests/integration/test_journal_completion_process.py @@ -0,0 +1,145 @@ +"""Run only after the concurrent FULL has ended: real process termination boundaries.""" + +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels +from tests.integration.test_workspace_runtime_exclusivity_baseline import ( + _child_env, # pyright: ignore[reportPrivateUsage] +) + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.workspace_owner import WorkspaceInUse +from thoth.apps.runtime import AppRuntime + +ROOT = Path(__file__).resolve().parents[2] +CHILD = r""" +import asyncio, sys, time +from pathlib import Path +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request,value +from tests.integration.test_operation_journal_failure_boundaries import NoModels +from thoth.adapters.storage.events import SqliteEventStore + +async def main(workspace, marker, phase): + model=NoModels() + runtime=create_runtime(workspace,model_resolver=model) + value(await runtime.bus.dispatch(request('project/create','create',{ + 'project_id':'project:j-process','name':'Original','cutoff_at':'2026-10-01T00:00:00Z'}))) + command=request('project/metadata/update','gap',{ + 'project_id':'project:j-process','expected_revision':0,'name':'Saved once'}) + ticket=runtime.bus.claim(command) + identifier=ticket.operation.operation_id + point_insert=SqliteEventStore.insert_checkpoint + event_insert=SqliteEventStore.insert_event + def ready(): + assert model.calls==0 + marker.write_text('ready',encoding='utf-8') + time.sleep(120) + raise AssertionError('parent did not terminate the test child') + if phase=='repair-checkpoint': + def initial_gap(self,point): + if point.operation_id==identifier:raise RuntimeError('initial gap') + point_insert(self,point) + SqliteEventStore.insert_checkpoint=initial_gap + try:await runtime.bus.execute(ticket) + except RuntimeError as exc:assert str(exc)=='initial gap' + else:raise AssertionError('initial gap not reached') + def checkpoint(self,point): + if point.operation_id==identifier and phase=='normal-checkpoint':ready() + point_insert(self,point) + if point.operation_id==identifier and phase=='repair-checkpoint':ready() + def event(self,item): + event_insert(self,item) + if (item.operation_id==identifier + and item.event_type=='project/metadata/updated' + and phase=='normal-notification'): + ready() + SqliteEventStore.insert_checkpoint=checkpoint + SqliteEventStore.insert_event=event + if phase=='repair-checkpoint':await runtime.bus.dispatch(command) + else:await runtime.bus.execute(ticket) + raise AssertionError('termination point not reached') +asyncio.run(main(Path(sys.argv[1]),Path(sys.argv[2]),sys.argv[3])) +""" + + +def _open(workspace: Path, model: NoModels) -> AppRuntime: + """Use the existing bounded Windows lock-release retry from the owner recovery tests.""" + deadline = time.monotonic() + 10 + while True: + try: + return create_runtime(workspace, model_resolver=model) + except WorkspaceInUse: + if time.monotonic() > deadline: + raise + time.sleep(0.02) + + +@pytest.mark.parametrize("phase", ["normal-checkpoint", "normal-notification", "repair-checkpoint"]) +async def test_process_exit_preserves_result_and_atomic_supplement( + tmp_path: Path, phase: str +) -> None: + workspace, marker, home = tmp_path / "workspace", tmp_path / "ready", tmp_path / "home" + home.mkdir() + log = tmp_path / "child.log" + with log.open("wb") as sink: + child = subprocess.Popen( + [sys.executable, "-B", "-c", CHILD, str(workspace), str(marker), phase], + cwd=ROOT, + env=_child_env(home), + stdout=sink, + stderr=sink, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + try: + deadline = time.monotonic() + 60 + while not marker.exists() and time.monotonic() < deadline: + assert child.poll() is None, log.read_text(errors="replace")[-2000:] + time.sleep(0.025) + assert marker.exists(), "child did not reach the requested journal boundary" + child.kill() + child.wait(timeout=10) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=10) + model = NoModels() + runtime = _open(workspace, model) + try: + with runtime.ledger.engine.connect() as connection: + row = connection.exec_driver_sql( + "SELECT operation_id,state FROM operations WHERE idempotency_key='gap'" + ).one() + identifier = str(row[0]) + assert row[1] == "SUCCEEDED" + journal = SqliteEventStore(runtime.ledger.engine) + assert len(journal.list_checkpoints(identifier)) == ( + 1 if phase == "normal-notification" else 0 + ) + assert len(journal.list_events(identifier)) == (2 if phase == "normal-notification" else 1) + before = snapshot(runtime.ledger.engine) + command = request( + "project/metadata/update", + "gap", + { + "project_id": "project:j-process", + "expected_revision": 0, + "name": "Saved once", + }, + ) + assert value(await runtime.bus.dispatch(command))["revision"] == 1 + after = snapshot(runtime.ledger.engine) + assert all(before[t] == after[t] for t in before if t not in {"events", "checkpoints"}) + assert len(journal.list_events(identifier)) == 3 + assert len(journal.list_checkpoints(identifier)) == 1 + assert value(await runtime.bus.dispatch(command))["revision"] == 1 + assert snapshot(runtime.ledger.engine) == after and model.calls == 0 + finally: + runtime.close() diff --git a/tests/integration/test_journal_completion_replay.py b/tests/integration/test_journal_completion_replay.py new file mode 100644 index 0000000..44af230 --- /dev/null +++ b/tests/integration/test_journal_completion_replay.py @@ -0,0 +1,173 @@ +"""Measured LOCAL success gaps are repaired once without rerunning their effects.""" + +from pathlib import Path + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_operation_journal_failure_boundaries import NoModels + +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.operations import SqliteOperationStore +from thoth.domain.event import Checkpoint, EventRecord +from thoth.protocol.bus import DispatchTicket + +NOTIFICATIONS = { + "project/metadata/update": ["project/metadata/updated"], + "project/archive": ["project/archived", "project/statusChanged"], + "project/restore": ["project/statusChanged"], +} + + +class InjectedTailFailure(RuntimeError): + pass + + +@pytest.mark.parametrize( + ("method", "gap"), + [ + *( + ("project/metadata/update", gap) + for gap in [ + "checkpoint-before", + "checkpoint-after", + "notification-before", + "notification-after", + "success-before", + "success-after", + ] + ), + ("project/archive", "notification-before"), + ("project/archive", "second-notification-before"), + ("project/archive", "second-notification-after"), + ("project/restore", "notification-before"), + ], +) +async def test_gap_repair_preserves_result_and_existing_rows( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: str, gap: str +) -> None: + model = NoModels() + runtime = create_runtime(tmp_path, model_resolver=model) + command_fields: dict[str, object] = {"project_id": "project:j", "expected_revision": 0} + try: + value( + await runtime.bus.dispatch( + request( + "project/create", + "create", + { + "project_id": "project:j", + "name": "Original", + "cutoff_at": "2026-10-01T00:00:00Z", + }, + ) + ) + ) + if method == "project/metadata/update": + command_fields["name"] = "Saved once" + if method == "project/restore": + value( + await runtime.bus.dispatch( + request("project/archive", "prepare-archive", command_fields) + ) + ) + command_fields["expected_revision"] = 1 + command = request(method, "gap", command_fields) + ticket = runtime.bus.claim(command) + assert isinstance(ticket, DispatchTicket) + identifier = ticket.operation.operation_id + checkpoint_insert = SqliteEventStore.insert_checkpoint + event_insert = SqliteEventStore.insert_event + hits: list[str] = [] + + def checkpoint(self: SqliteEventStore, point: Checkpoint) -> None: + hit = point.operation_id == identifier and gap.startswith("checkpoint-") + if hit and gap.endswith("before"): + hits.append(gap) + raise InjectedTailFailure(gap) + checkpoint_insert(self, point) + if hit: + hits.append(gap) + raise InjectedTailFailure(gap) + + def event(self: SqliteEventStore, item: EventRecord) -> None: + index = 1 if gap.startswith("second-notification-") else 0 + selected = ( + "operation.succeeded" + if gap.startswith("success-") + else NOTIFICATIONS[method][index] + ) + hit = ( + item.operation_id == identifier + and item.event_type == selected + and not gap.startswith("checkpoint-") + ) + if hit and gap.endswith("before"): + hits.append(gap) + raise InjectedTailFailure(gap) + event_insert(self, item) + if hit: + hits.append(gap) + raise InjectedTailFailure(gap) + + with monkeypatch.context() as patch: + patch.setattr(SqliteEventStore, "insert_checkpoint", checkpoint) + patch.setattr(SqliteEventStore, "insert_event", event) + with pytest.raises(InjectedTailFailure): + await runtime.bus.execute(ticket) + assert hits == [gap] + store = SqliteOperationStore(runtime.ledger.engine) + original = store.read(identifier) + assert ( + original is not None + and original.result is not None + and original.completed_at is not None + ) + original_bytes = original.model_dump_json() + journal = SqliteEventStore(runtime.ledger.engine) + old_events = { + item.event_id: item.model_dump_json() for item in journal.list_events(identifier) + } + old_points = { + item.checkpoint_id: item.model_dump_json() + for item in journal.list_checkpoints(identifier) + } + before = snapshot(runtime.ledger.engine) + repaired = await runtime.bus.dispatch(command) + assert value(repaired) == original.result + stored = store.read(identifier) + assert stored is not None and stored.model_dump_json() == original_bytes + after = snapshot(runtime.ledger.engine) + assert all( + before[name] == after[name] for name in before if name not in {"events", "checkpoints"} + ) + events = journal.list_events(identifier) + assert [item.event_type for item in events] == [ + "operation.started", + *NOTIFICATIONS[method], + "operation.succeeded", + ] + points = journal.list_checkpoints(identifier) + assert len(points) == 1 + for item in events: + if item.event_id in old_events: + assert item.model_dump_json() == old_events[item.event_id] + else: + assert item.payload["supplemented"] is True + assert item.payload["supplemented_for"] == original.completed_at.isoformat() + for point in points: + if point.checkpoint_id in old_points: + assert point.model_dump_json() == old_points[point.checkpoint_id] + else: + assert point.payload["supplemented"] is True + assert point.payload["supplemented_for"] == original.completed_at.isoformat() + assert await runtime.bus.dispatch(command) == repaired + assert snapshot(runtime.ledger.engine) == after + runtime.close() + runtime = create_runtime(tmp_path, model_resolver=model) + assert await runtime.bus.dispatch(command) == repaired + assert snapshot(runtime.ledger.engine) == after + assert model.calls == 0 + finally: + runtime.close() diff --git a/tests/integration/test_m3_hypothesis_first_write_reachability.py b/tests/integration/test_m3_hypothesis_first_write_reachability.py index 64d6bcb..fa284ff 100644 --- a/tests/integration/test_m3_hypothesis_first_write_reachability.py +++ b/tests/integration/test_m3_hypothesis_first_write_reachability.py @@ -178,9 +178,13 @@ def observe_window(assessment: Any, validation: str) -> None: save(case, evidence) @contextmanager - def accept(service: Any, spec: Any, bundle: Any) -> Generator[Any]: + def accept( + service: Any, spec: Any, bundle: Any, *, defer_rejection: bool = False + ) -> Generator[Any]: if not active(): - with originals["accept"](service, spec, bundle) as artifact: + with originals["accept"]( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: yield artifact return evidence["ambient_at_entry"] = vars(transaction_module)["_AMBIENT"].get() is not None @@ -190,7 +194,9 @@ def accept(service: Any, spec: Any, bundle: Any) -> Generator[Any]: evidence["before_phase"] = snapshot(runtime.ledger.engine) evidence["trace"].append("before_original_accept") try: - with originals["accept"](service, spec, bundle) as artifact: + with originals["accept"]( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: evidence["trace"].append("original_accept_entered") yield artifact except BaseException as exc: diff --git a/tests/integration/test_m3_substantive_appraisal_atomicity.py b/tests/integration/test_m3_substantive_appraisal_atomicity.py index b8f5533..ccb4f0c 100644 --- a/tests/integration/test_m3_substantive_appraisal_atomicity.py +++ b/tests/integration/test_m3_substantive_appraisal_atomicity.py @@ -81,9 +81,13 @@ def active() -> bool: ) == ("thread/input", key, project) @contextmanager - def observe_accept(service: SandboxService, spec: Any, bundle: Any) -> Generator[Any]: + def observe_accept( + service: SandboxService, spec: Any, bundle: Any, *, defer_rejection: bool = False + ) -> Generator[Any]: if not active() or spec.project_id != project: - with original_accept(service, spec, bundle) as artifact: + with original_accept( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: yield artifact return phase_attempt.append(spec.attempt_id) @@ -99,7 +103,9 @@ def observe_accept(service: SandboxService, spec: Any, bundle: Any) -> Generator if not evidence["ambient_at_entry"]: evidence["before"] = snapshot(runtime.ledger.engine) try: - with original_accept(service, spec, bundle) as artifact: + with original_accept( + service, spec, bundle, defer_rejection=defer_rejection + ) as artifact: evidence["trace"].append("original-entered") yield artifact except BaseException as exc: diff --git a/tests/integration/test_m4_memory_control_multiwrite_observation.py b/tests/integration/test_m4_memory_control_multiwrite_observation.py index 4f2b110..16c3e29 100644 --- a/tests/integration/test_m4_memory_control_multiwrite_observation.py +++ b/tests/integration/test_m4_memory_control_multiwrite_observation.py @@ -1,19 +1,17 @@ -"""Step 0 of 96b: what the second write of a two-write MEMORY control route leaves behind. +"""Two-write MEMORY observations, updated to the approved P1 atomic-pair contract. memory/validate (GATE then CANDIDATE revision), memory/changeSet/propose (MEMORY_ENTRY_CANDIDATE then CHANGE_SET_PROPOSAL) and memory/retire/propose (RETIRE_CANDIDATE then -CHANGE_SET_PROPOSAL) call ControlRecordService.create twice. Each create is its own -SqliteControlRecordStore.append commit, the bus does not wrap a handler in a ledger transaction, and -the handler awaits nothing between the two writes. This file observes the current contract and fixes -nothing; it is test-only. +CHANGE_SET_PROPOSAL) call ControlRecordService.create twice. P1 wraps each pair in the handler's +short ledger transaction; the handler awaits nothing between these writes. The second append of the target request is made to fail with an injected error, either before it -writes ("before": the state is exactly what the first write left) or after it wrote ("after": both -rows are committed but the caller sees an error). Everything else is observed from the real runtime: -the rows, the other tables, the sealed failed operation, a same-key replay, a reopened runtime, a -new-key retry and the reads a later request would make. validate also gets one simulated competing +writes ("before") or after it wrote ("after"). Both now roll back the whole pair. Before P1, these +faults left one or two committed rows; the V-P1 evidence retains that baseline. The real runtime +checks all tables, the failed operation, same-key replay, runtime reopening, new-key retry, +and later reads. validate also gets one simulated competing writer (a second revision of the same record between the read and the append), which is what another -process could do; in one process it cannot happen. +process could previously do; this nested injection does not prove cross-process concurrency safety. Evidence files: outputs/refactor-step96-20261015/step0-multiwrite-*.json. """ @@ -300,7 +298,7 @@ async def test_control_route_success_writes_the_two_rows_in_source_order( @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["before", "after"]) @pytest.mark.parametrize("route", ROUTES) -async def test_second_control_write_failure_leaves_the_first_row_and_a_failed_operation( +async def test_second_control_write_failure_rolls_back_both_rows_and_keeps_failed_operation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, route: str, mode: str ) -> None: workspace = tmp_path / "workspace" @@ -320,13 +318,9 @@ async def test_second_control_write_failure_leaves_the_first_row_and_a_failed_op fault_after = snapshot(runtime.ledger.engine) changes = table_changes(before, fault_after) added = rows_after[len(rows_before) :] - expected_added = 1 if mode == "before" else 2 - assert [item["type"] for item in added] == [t for t, _ in EXPECTED_ORDER[route]][ - :expected_added - ] - assert set(domain_changes(changes, before, fault_after, request(method, key, payload))) == { - "control_records" - } + assert added == [] + assert rows_after == rows_before + assert domain_changes(changes, before, fault_after, request(method, key, payload)) == {} reads_after_fault = await later_reads(runtime, route, candidate_id, "fault") runtime.close() @@ -381,16 +375,12 @@ async def test_second_control_write_failure_leaves_the_first_row_and_a_failed_op ) assert [item["type"] for item in retry_added] == [t for t, _ in EXPECTED_ORDER[route]] if route == "validate": - # Current contract: the first row is a visible VALIDATED gate while the candidate - # has not advanced ("before"), or the candidate advanced although the caller saw an - # error ("after"); a new-key retry then adds a second gate with the same digest. - assert reads_after_fault["candidate_state"] == ( - "CANDIDATE" if mode == "before" else "VALIDATED" - ) - assert len(reads_after_fault["gates"]) == 1 - assert reads_after_fault["gates"][0]["state"] == "VALIDATED" + # Approved P1 contract: neither gate nor candidate revision leaks on either fault. + assert reads_after_fault == reads_before + assert reads_after_fault["candidate_state"] == "CANDIDATE" + assert reads_after_fault["gates"] == [] assert reads_after_retry["candidate_state"] == "VALIDATED" - assert len(reads_after_retry["gates"]) == 2 + assert len(reads_after_retry["gates"]) == 1 assert len({gate["gate_digest"] for gate in reads_after_retry["gates"]}) == 1 else: assert reads_after_fault == reads_before diff --git a/tests/integration/test_operation_journal_failure_boundaries.py b/tests/integration/test_operation_journal_failure_boundaries.py index 3758168..52060f1 100644 --- a/tests/integration/test_operation_journal_failure_boundaries.py +++ b/tests/integration/test_operation_journal_failure_boundaries.py @@ -284,6 +284,7 @@ def fail_checkpoint(store: SqliteEventStore, checkpoint: Checkpoint) -> None: "journal_after_failure": journal_status(after, identifier), "journal_after_reopen_retry": journal_status(final, identifier), "no_writes_on_retries": after == retried == final, + "no_writes_after_completion": retried == final, "snapshots": { "before": snapshot_summary(before), "after": snapshot_summary(after), @@ -292,8 +293,8 @@ def fail_checkpoint(store: SqliteEventStore, checkpoint: Checkpoint) -> None: }, "changed_tables": [name for name in before if before[name] != after[name]], "journal_disposition": ( - "OBSERVED_POST_COMPLETION_GAP" - if mode == "checkpoint" and journal_status(final, identifier)["missing"] + "SUPPLEMENTED_POST_COMPLETION_GAP" + if mode == "checkpoint" and not journal_status(final, identifier)["missing"] else "OBSERVED" ), } @@ -312,7 +313,26 @@ def fail_checkpoint(store: SqliteEventStore, checkpoint: Checkpoint) -> None: f"running_tasks={live_tasks, reopened_tasks}" ) else: - assert after == retried == final + if mode == "checkpoint": + assert journal_status(after, identifier)["missing"] + assert journal_status(final, identifier)["missing"] == [] + assert retried == final + assert all( + after[name] == final[name] + for name in after + if name not in {"events", "checkpoints"} + ) + added_events = [json.loads(row) for row in set(final["events"]) - set(after["events"])] + added_points = [ + json.loads(row) for row in set(final["checkpoints"]) - set(after["checkpoints"]) + ] + assert len(added_events) == 2 and len(added_points) == 1 + for row in (*added_events, *added_points): + payload = json.loads(row["payload_json"]) + assert payload["supplemented"] is True + assert payload["supplemented_for"] == operation["completed_at"] + else: + assert after == retried == final assert same_runtime.result == after_reopen.result stored_result = operation["result_json"] assert isinstance(stored_result, str) diff --git a/tests/integration/test_p_memory_atomicity.py b/tests/integration/test_p_memory_atomicity.py new file mode 100644 index 0000000..e5ac9e7 --- /dev/null +++ b/tests/integration/test_p_memory_atomicity.py @@ -0,0 +1,150 @@ +"""Real RPC failures at either append must publish neither memory control record.""" + +from pathlib import Path +from typing import Any + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.test_m4_memory_control_multiwrite_observation import ( + EXPECTED_ORDER, + ROUTES, + call, + domain_changes, + later_reads, + prepare, + request, + rows, + table_changes, +) + +from thoth.adapters.storage.control_record import SqliteControlRecordStore +from thoth.application.services.control_record_service import ControlRecordService +from thoth.apps.runtime import create_runtime +from thoth.domain.control_record import ControlRecord +from thoth.protocol.deferred import current_operation + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("write_number", [1, 2]) +@pytest.mark.parametrize("after_write", [False, True]) +async def test_append_fault_rolls_back_pair_and_replay_keeps_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + route: str, + write_number: int, + after_write: bool, +) -> None: + workspace = tmp_path / "workspace" + runtime, method, payload, candidate_id = await prepare(workspace, route) + key = f"p1-{route}-{write_number}-{after_write}" + original = SqliteControlRecordStore.append + hits: list[str] = [] + + def append(store: SqliteControlRecordStore, record: ControlRecord) -> None: + operation = current_operation.get() + if operation is None or operation.idempotency_key != key: + original(store, record) + return + hits.append(record.record_type) + if len(hits) == write_number: + if after_write: + original(store, record) + raise RuntimeError("P1 injected append failure") + original(store, record) + + try: + reads_before = await later_reads(runtime, route, candidate_id, "p1-before") + rows_before = rows(runtime) + before = snapshot(runtime.ledger.engine) + command = request(method, key, payload) + with monkeypatch.context() as patch: + patch.setattr(SqliteControlRecordStore, "append", append) + failed = await runtime.bus.dispatch(command) + assert failed.error is not None + assert hits == [kind for kind, _ in EXPECTED_ORDER[route]][:write_number] + after = snapshot(runtime.ledger.engine) + assert rows(runtime) == rows_before + assert domain_changes(table_changes(before, after), before, after, command) == {} + assert await later_reads(runtime, route, candidate_id, "p1-after") == reads_before + runtime.close() + runtime = create_runtime(workspace) + assert rows(runtime) == rows_before + before_replay = snapshot(runtime.ledger.engine) + replay = await runtime.bus.dispatch(command) + assert replay.error == failed.error + assert snapshot(runtime.ledger.engine) == before_replay + await call(runtime, method, key + "-fresh", payload) + added = rows(runtime)[len(rows_before) :] + assert [row["type"] for row in added] == [kind for kind, _ in EXPECTED_ORDER[route]] + finally: + runtime.close() + + +async def test_bound_helper_looks_up_create_after_handler_construction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, method, payload, _ = await prepare(tmp_path / "workspace", "validate") + key = "p1-create-monkeypatch-after-construction" + original = ControlRecordService.create + hits: list[str] = [] + + def create( + service: ControlRecordService, + *, + project_id: str, + namespace: str, + record_type: str, + state: str, + payload: dict[str, object], + record_id: str | None = None, + ) -> ControlRecord: + operation = current_operation.get() + if operation is not None and operation.idempotency_key == key: + hits.append(record_type) + return original( + service, + project_id=project_id, + namespace=namespace, + record_type=record_type, + state=state, + payload=payload, + record_id=record_id, + ) + + try: + monkeypatch.setattr(ControlRecordService, "create", create) + await call(runtime, method, key, payload) + assert hits == ["GATE", "CANDIDATE"] + finally: + runtime.close() + + +@pytest.mark.parametrize("route", ROUTES) +async def test_partial_first_append_is_not_visible_to_independent_reader( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + runtime, method, payload, _ = await prepare(tmp_path / "workspace", route) + key = f"p1-reader-{route}" + original = SqliteControlRecordStore.append + first_views: list[list[dict[str, Any]]] = [] + seen: list[str] = [] + before = rows(runtime) + + def append(store: SqliteControlRecordStore, record: ControlRecord) -> None: + original(store, record) + operation = current_operation.get() + if operation is not None and operation.idempotency_key == key: + seen.append(record.record_type) + if len(seen) == 1: + # rows() deliberately opens another connection, bypassing ambient UoW. + first_views.append(rows(runtime)) + + try: + with monkeypatch.context() as patch: + patch.setattr(SqliteControlRecordStore, "append", append) + await call(runtime, method, key, payload) + assert seen == [kind for kind, _ in EXPECTED_ORDER[route]] + assert first_views == [before] + assert len(rows(runtime)) == len(before) + 2 + finally: + runtime.close() diff --git a/tests/integration/test_p_r2_failure_atomicity.py b/tests/integration/test_p_r2_failure_atomicity.py new file mode 100644 index 0000000..11bba5a --- /dev/null +++ b/tests/integration/test_p_r2_failure_atomicity.py @@ -0,0 +1,231 @@ +"""Prepared R2 failures publish hold/reject and final completion/candidate atomically.""" + +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path +from typing import Any, cast + +import pytest +from tests.atomicity.harness import snapshot +from tests.integration.p_r2_failure_helpers import ambient, capture_exit, revise_hypothesis, save +from tests.integration.test_a02_autonomous_acquisition import StaticModelResolver +from tests.integration.test_a04_r2_closed_loop import A04R2Model +from tests.integration.test_hypothesis_prediction_outcome_cycle import ( + MeasurementSandbox, + prepare_measurement, + request, + value, +) + +from thoth.application.services.r2_closed_loop import R2ClosedLoopCoordinator +from thoth.application.services.r2_test_lifecycle import R2TestLifecycle +from thoth.application.services.sandbox_service import SandboxService +from thoth.apps.runtime import create_runtime +from thoth.domain.sandbox import SandboxErrorCode, SandboxExecutionState, SandboxFailure +from thoth.ports.model import ModelPort + + +@pytest.mark.parametrize( + "path", ["stale", "accept_success", "accept_failure", "assessment", "finalization"] +) +@pytest.mark.parametrize("after_write", [False, True]) +@pytest.mark.parametrize("write_number", [1, 2]) +async def test_prepared_rejection_pair_rolls_back_on_write_fault( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, path: str, after_write: bool, write_number: int +) -> None: + states = (SandboxExecutionState.FAILED,) if path == "accept_failure" else () + runtime, sandbox, project, thread, _ = await prepare_measurement( + tmp_path, monkeypatch, states=states + ) + evidence: dict[str, Any] = {"path": path, "trace": [], "faults": 0, "await_ambient": []} + originals = {"hold": R2TestLifecycle.hold, "reject": SandboxService.reject_observation} + + async def during_run(spec: Any) -> None: + evidence["await_ambient"].append(ambient()) + if path == "stale": + await revise_hypothesis(runtime, project, spec) + + sandbox.on_result = during_run + + def pair_call(name: str, instance: Any, *args: Any, **kwargs: Any) -> Any: + if not evidence["trace"]: + evidence["before_phase"] = snapshot(runtime.ledger.engine) + evidence["trace"].append(name) + target = len(evidence["trace"]) == write_number + if target and not after_write: + evidence["faults"] += 1 + raise RuntimeError("P2 second pair write fault") + result = originals[name](instance, *args, **kwargs) + if target: + evidence["faults"] += 1 + raise RuntimeError("P2 second pair write fault") + return result + + def hold(instance: Any, *args: Any, **kwargs: Any) -> Any: + return pair_call("hold", instance, *args, **kwargs) + + def reject(instance: Any, *args: Any, **kwargs: Any) -> Any: + return pair_call("reject", instance, *args, **kwargs) + + original_basis = SandboxService._validate_admission_basis # pyright: ignore[reportPrivateUsage] + + def basis(instance: Any, *args: Any, **kwargs: Any) -> Any: + if sandbox.seen_specs: + raise SandboxFailure(SandboxErrorCode.INPUT_INVALID, "P2 admission changed") + return original_basis(instance, *args, **kwargs) + + def invalid_assessment(*args: Any, **kwargs: Any) -> Any: + raise ValueError("RESEARCH_TEST_ASSESSMENT_REJECTED") + + def invalid_completion(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError("P2 finalization failed") + + try: + command = request("thread/input", "p2-pair", {"project_id": project, "thread_id": thread}) + with monkeypatch.context() as patch: + capture_exit(patch, runtime, evidence) + patch.setattr(R2TestLifecycle, "hold", hold) + patch.setattr(SandboxService, "reject_observation", reject) + if path.startswith("accept_"): + patch.setattr(SandboxService, "_validate_admission_basis", basis) + elif path == "assessment": + patch.setattr(R2TestLifecycle, "prepare_assessments", invalid_assessment) + elif path == "finalization": + patch.setattr(R2TestLifecycle, "complete", invalid_completion) + response = await runtime.bus.dispatch(command) + evidence["response"] = response.model_dump(mode="json") + save(f"pair-{path}-{write_number}-{after_write}", evidence) + assert evidence["faults"] == 1 + expected = ["hold", "reject"] if path == "stale" else ["reject", "hold"] + assert evidence["trace"] == expected[:write_number] + assert evidence["await_ambient"] == [False] + assert evidence["ambient_after_phase"] is False + assert response.error is not None + assert evidence["after_phase"] == evidence["before_phase"] + terminal = snapshot(runtime.ledger.engine) + replay = await runtime.bus.dispatch(command) + assert replay.error == response.error + assert snapshot(runtime.ledger.engine) == terminal + assert len(sandbox.seen_specs) == 1 + finally: + runtime.close() + + +@pytest.mark.parametrize("state", [SandboxExecutionState.FAILED, SandboxExecutionState.TIMED_OUT]) +@pytest.mark.parametrize("after_candidate", [False, True]) +@pytest.mark.parametrize("sandbox_error", [False, True]) +async def test_final_prepared_completion_and_candidate_roll_back_together( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + state: SandboxExecutionState, + after_candidate: bool, + sandbox_error: bool, +) -> None: + runtime, sandbox, project, thread, _ = await prepare_measurement( + tmp_path, monkeypatch, states=(state,), max_transient_retries=0 + ) + evidence: dict[str, Any] = {"complete_hits": 0, "candidate_hits": 0, "await_ambient": []} + original_accept = SandboxService.accepting_observation + original_complete = R2TestLifecycle.complete + original_failure = R2ClosedLoopCoordinator._failure_execution # pyright: ignore[reportPrivateUsage] + + async def during_run(spec: Any) -> None: + evidence["await_ambient"].append(ambient()) + + sandbox.on_result = during_run + + @contextmanager + def accept(instance: Any, spec: Any, bundle: Any, **kwargs: Any) -> Generator[Any]: + evidence["before_phase"] = snapshot(runtime.ledger.engine) + with original_accept(instance, spec, bundle, **kwargs) as artifact: + yield artifact + + def complete(instance: Any, *args: Any, **kwargs: Any) -> Any: + evidence["complete_hits"] += 1 + return original_complete(instance, *args, **kwargs) + + def failure(instance: Any, **kwargs: Any) -> Any: + evidence["candidate_hits"] += 1 + evidence["has_test_completion"] = kwargs.get("test_completion") is not None + if after_candidate: + result = original_failure(instance, **kwargs) + evidence["candidate_projection"] = result.projection + if sandbox_error: + raise SandboxFailure( + SandboxErrorCode.INPUT_INVALID, "P2 candidate error, not admission" + ) + raise RuntimeError("P2 candidate publication fault") + + try: + command = request("thread/input", "p2-final", {"project_id": project, "thread_id": thread}) + with monkeypatch.context() as patch: + capture_exit(patch, runtime, evidence) + patch.setattr(SandboxService, "accepting_observation", accept) + patch.setattr(R2TestLifecycle, "complete", complete) + patch.setattr(R2ClosedLoopCoordinator, "_failure_execution", failure) + response = await runtime.bus.dispatch(command) + evidence["response"] = response.model_dump(mode="json") + save(f"final-{state.value}-{after_candidate}-{sandbox_error}", evidence) + assert evidence["complete_hits"] == evidence["candidate_hits"] == 1 + assert evidence["has_test_completion"] is True + assert evidence["await_ambient"] == [False] + assert evidence["ambient_after_phase"] is False + assert response.error is not None + assert evidence["after_phase"] == evidence["before_phase"] + terminal = snapshot(runtime.ledger.engine) + replay = await runtime.bus.dispatch(command) + assert replay.error == response.error + assert snapshot(runtime.ledger.engine) == terminal + assert len(sandbox.seen_specs) == 1 + runtime.close() + runtime = create_runtime( + tmp_path / "allowed", + sandbox_adapter=sandbox, + model_resolver=StaticModelResolver(cast(ModelPort, A04R2Model())), + ) + assert snapshot(runtime.ledger.engine) == terminal + replay = await runtime.bus.dispatch(command) + assert replay.error == response.error + assert snapshot(runtime.ledger.engine) == terminal + again = value( + await runtime.bus.dispatch( + request("thread/input", "p2-new-key", {"project_id": project, "thread_id": thread}) + ) + )["r2_closed_loop"] + assert again["reason_code"] == "RESEARCH_TEST_RESULT_RECONCILIATION_REQUIRED" + assert again["pending_result"]["exits"] == ["CLEAR_AND_RERUN"] + assert len(sandbox.seen_specs) == 1 + finally: + runtime.close() + + +async def test_retry_awaits_stay_outside_transactions( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runtime, sandbox, project, thread, _ = await prepare_measurement( + tmp_path, + monkeypatch, + states=(SandboxExecutionState.TIMED_OUT, SandboxExecutionState.SUCCEEDED), + max_transient_retries=1, + ) + original = MeasurementSandbox.run + boundaries: list[tuple[str, bool]] = [] + + async def run(adapter: MeasurementSandbox, spec: Any) -> Any: + boundaries.append(("before", ambient())) + result = await original(adapter, spec) + boundaries.append(("after", ambient())) + return result + + try: + monkeypatch.setattr(MeasurementSandbox, "run", run) + result = value( + await runtime.bus.dispatch( + request("thread/input", "p2-retry", {"project_id": project, "thread_id": thread}) + ) + ) + assert result["r2_closed_loop"]["terminal_state"] == "COMPLETED" + assert boundaries == [("before", False), ("after", False)] * 2 + assert len(sandbox.seen_specs) == 2 + finally: + runtime.close() diff --git a/tests/integration/test_p_reference_failure_notice.py b/tests/integration/test_p_reference_failure_notice.py new file mode 100644 index 0000000..8c2324b --- /dev/null +++ b/tests/integration/test_p_reference_failure_notice.py @@ -0,0 +1,256 @@ +"""A real post-research reference rejection must describe only readable saved research.""" + +import json +import os +from pathlib import Path +from typing import Any + +import pytest +from tests.integration.reference_helpers import reference_harness +from tests.integration.storage_coverage_helpers import request, value +from tests.integration.test_a02_autonomous_acquisition import DynamicA02Model, StaticModelResolver +from tests.integration.test_research_request_v2 import ControlledResearchModel + +from thoth.application.commands.reference import ReferenceThreadEntry +from thoth.application.commands.thread_analysis import ThreadAnalysisCommandHandlers +from thoth.application.services.revision_service import CommitResult +from thoth.domain.behavior_execution import BehaviorExecutionRecord +from thoth.domain.reference import ReferenceError +from thoth.protocol.deferred import current_operation + + +def save(name: str, evidence: dict[str, Any]) -> None: + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/refactor-p-20261022" + / os.environ.get("THOTH_P_RUN_LABEL", "p3-direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{name}.json").write_text( + json.dumps(evidence, indent=2, default=str) + "\n", encoding="utf-8" + ) + + +def install_models(monkeypatch: pytest.MonkeyPatch, version: int) -> list[str]: + controlled = ControlledResearchModel(one=True) + calls: list[str] = [] + dynamic = DynamicA02Model.structured + structured = ControlledResearchModel.structured + + async def legacy_model(instance: DynamicA02Model, call: Any) -> Any: + calls.append(call.role.value) + return await dynamic(instance, call) + + async def controlled_model(instance: ControlledResearchModel, call: Any) -> Any: + calls.append(call.role.value) + return await structured(instance, call) + + def resolve( + instance: StaticModelResolver, *, provider: str, model: str | None + ) -> ControlledResearchModel: + return controlled + + monkeypatch.setattr(DynamicA02Model, "structured", legacy_model) + monkeypatch.setattr(ControlledResearchModel, "structured", controlled_model) + if version == 2: + monkeypatch.setattr(StaticModelResolver, "resolve", resolve) + return calls + + +@pytest.mark.parametrize("version", [1, 2]) +async def test_real_reference_rejection_reports_saved_research_in_both_contracts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, version: int +) -> None: + async with reference_harness(tmp_path) as h: + calls = install_models(monkeypatch, version) + captured: dict[str, Any] = {} + original = ReferenceThreadEntry._reference # pyright: ignore[reportPrivateUsage] + + async def reference(entry: ReferenceThreadEntry, selected: Any, result: Any) -> Any: + op = current_operation.get() + assert op is not None + captured["operation_id"] = op.operation_id + captured["delegate_result"] = result + return await original(entry, selected, result) + + monkeypatch.setattr(ReferenceThreadEntry, "_reference", reference) + command = request( + "thread/input", + "p3-missing-inquiry", + { + "project_id": h.project, + "thread_id": h.thread, + "instruction": "Compare the saved source measurements.", + **({"contract_version": 2} if version == 2 else {}), + "reference_request": {**h.request, "inquiry_id": "missing:p3-inquiry"}, + }, + ) + response = await h.runtime.bus.dispatch(command) + await h.runtime.bus.drain() + assert captured, response + operation = h.runtime.bus.read_operation(captured["operation_id"]) + assert operation is not None and operation.error is not None + error = operation.error + commit = CommitResult.model_validate(captured["delegate_result"]["commit"]) + assert commit.receipt in h.runtime.ledger.read_receipts(h.project) + assert commit.committed_revision_ids + readable: list[dict[str, Any]] = [] + for identifier in commit.committed_revision_ids: + revision = h.runtime.ledger.read_revision_by_id(h.project, identifier) + assert revision is not None + read = value( + await h.runtime.bus.dispatch( + request( + "revision/read", + "p3-read-" + identifier, + {"project_id": h.project, "revision_digest": revision.revision_digest}, + ) + ) + ) + content = value( + await h.runtime.bus.dispatch( + request( + "revision/content/read", + "p3-content-" + identifier, + {"project_id": h.project, "content_digest": read["content_digest"]}, + ) + ) + ) + readable.append({"revision": read, "content": content}) + current = value( + await h.runtime.bus.dispatch( + request( + "thread/read", + "p3-thread", + {"project_id": h.project, "thread_id": h.thread, "view": "FULL"}, + ) + ) + ) + captured.update(error=error, readable=readable, thread_read=current) + save(f"reference-v{version}", captured) + assert error["code"] == (-32030 if version == 1 else -32603) + data = error["data"] + assert isinstance(data, dict) + assert data["reason_code"] == "REFERENCE_INQUIRY_NOT_FOUND" + assert data.get("research_saved") is True + message = str(error["message"]) + assert "기준(reference) 단계만 실패했습니다" in message + assert "Only the reference step failed" in message + if version == 2: + failure = data["failure"] + assert isinstance(failure, dict) + primary = failure["primary"] + assert isinstance(primary, dict) + assert primary["exception_type"] == "RpcApplicationError" + manifest = current.get("current_result") or current.get("previous_result") + assert manifest is not None and manifest["completion"] != "TERMINAL" + assert "중간 기록" in message + before_replay_calls = len(calls) + assert before_replay_calls > 0 + replay = await h.runtime.bus.dispatch(command) + await h.runtime.bus.drain() + assert replay.error is not None + assert replay.error.model_dump(mode="json") == error + assert len(calls) == before_replay_calls + + +@pytest.mark.parametrize("version", [1, 2]) +@pytest.mark.parametrize("failure", ["before", "delegate", "unproven", "generic"]) +async def test_other_failures_never_claim_saved_reference_research( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, version: int, failure: str +) -> None: + async with reference_harness(tmp_path) as h: + calls = install_models(monkeypatch, version) + captured: dict[str, Any] = {} + original_entry = ReferenceThreadEntry.analyze + original_delegate = ThreadAnalysisCommandHandlers.analyze + + async def entry(owner: ReferenceThreadEntry, payload: Any) -> Any: + operation = current_operation.get() + assert operation is not None + captured["operation_id"] = operation.operation_id + captured["calls_at_reference"] = list(calls) + return await original_entry(owner, payload) + + async def delegate(owner: ThreadAnalysisCommandHandlers, payload: Any) -> Any: + if failure == "delegate": + raise ReferenceError("REFERENCE_DELEGATE_TEST_FAILURE") + result = await original_delegate(owner, payload) + if failure == "unproven": + result = {k: v for k, v in result.items() if k != "commit"} + return result + + async def generic(owner: ReferenceThreadEntry, selected: Any, result: Any) -> Any: + raise RuntimeError("private-test-marker-never-present-in-error") + + monkeypatch.setattr(ReferenceThreadEntry, "analyze", entry) + monkeypatch.setattr(ThreadAnalysisCommandHandlers, "analyze", delegate) + if failure == "generic": + monkeypatch.setattr(ReferenceThreadEntry, "_reference", generic) + selected = {**h.request, "inquiry_id": "missing:p3-inquiry"} + if failure == "before": + selected["expected_revision_digest"] = "0" * 64 + response = await h.input( + "p3-negative", + instruction="Compare the saved source measurements.", + reference_request=selected, + **({"contract_version": 2} if version == 2 else {}), + ) + await h.runtime.bus.drain() + assert captured, response + operation = h.runtime.bus.read_operation(captured["operation_id"]) + assert operation is not None and operation.error is not None + error = operation.error + data = error.get("data") or {} + assert isinstance(data, dict) + assert "research_saved" not in data + assert "저장됐습니다" not in str(error["message"]) + assert "were saved" not in str(error["message"]) + assert "private-test-marker" not in str(error) + if version == 2: + failure_record = data["failure"] + assert isinstance(failure_record, dict) + primary = failure_record["primary"] + assert isinstance(primary, dict) + assert primary["exception_type"] == ( + "RuntimeError" if failure == "generic" else "RpcApplicationError" + ) + if failure in {"before", "delegate"}: + assert calls == captured["calls_at_reference"] + assert calls == ( + ["RESEARCH_PLANNER", "EVIDENCE_RERANKER", "SEMANTIC_REVIEWER", "REVIEW_ADJUDICATOR"] + if version == 2 + else [] + ) + else: + assert calls + save(f"negative-{version}-{failure}", {"error": error, "model_roles": calls}) + + +@pytest.mark.parametrize("result", [{"queued": True}, {"unrelated_result": True}]) +async def test_queued_and_unassessed_results_do_not_enter_reference_notice( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, result: dict[str, bool] +) -> None: + async with reference_harness(tmp_path) as h: + + async def delegate(owner: ThreadAnalysisCommandHandlers, payload: Any) -> Any: + return result + + async def reference(owner: ReferenceThreadEntry, selected: Any, payload: Any) -> Any: + raise AssertionError("reference must not run for queued/unassessed input") + + monkeypatch.setattr(ThreadAnalysisCommandHandlers, "analyze", delegate) + monkeypatch.setattr(ReferenceThreadEntry, "_reference", reference) + response = await h.input("p3-no-reference", reference_request=h.request) + observed = value(response) + if result.get("queued") is True: + assert observed == result + else: + assert observed.keys() - result.keys() == {"behavior_execution"} + behavior = BehaviorExecutionRecord.model_validate(observed["behavior_execution"]) + assert (behavior.project_id, behavior.thread_id, behavior.state) == ( + h.project, + h.thread, + "COMPLETED", + ) + assert {k: v for k, v in observed.items() if k != "behavior_execution"} == result diff --git a/tests/integration/test_storage_clock_currentness.py b/tests/integration/test_storage_clock_currentness.py new file mode 100644 index 0000000..d736ca1 --- /dev/null +++ b/tests/integration/test_storage_clock_currentness.py @@ -0,0 +1,662 @@ +"""F-03b: real store appends select B after A despite a backwards or tied wall clock. + +No application runtime, server or model is constructed. Optional canonical-ledger stores are +exercised in their direct projection mode; this is not a claim that canonical head reads are broken. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import Engine, create_engine, literal_column, select +from tests.integration.scoped_runtime import create_runtime +from tests.integration.storage_coverage_helpers import request, value + +from thoth.adapters.runtime import SystemClock +from thoth.adapters.storage.action import SqliteActionStore +from thoth.adapters.storage.control_record import SqliteControlRecordStore +from thoth.adapters.storage.criterion_contract import SqliteCriterionContractStore +from thoth.adapters.storage.decision_object import SqliteDecisionObjectStore +from thoth.adapters.storage.events import SqliteEventStore +from thoth.adapters.storage.evidence_graph import SqliteEvidenceGraphStore +from thoth.adapters.storage.execution import SqliteExecutionStore +from thoth.adapters.storage.hypothesis import SqliteHypothesisStore +from thoth.adapters.storage.outcome import SqliteOutcomeStore +from thoth.adapters.storage.schema import metadata +from thoth.adapters.storage.thread_runtime import SqliteThreadRuntimeStore +from thoth.domain.action_full import ( + ActionPlanRecord, + ActionPortfolioRecord, + ActionRecord, + AuthorizationEnvelopeRecord, +) +from thoth.domain.base import DomainModel +from thoth.domain.control_record import ControlRecord +from thoth.domain.criterion_contract import CriterionContractRecord +from thoth.domain.decision_object_full import ( + DecisionObjectRecord, + ObjectAttentionRecord, + ObjectRelationRecord, +) +from thoth.domain.event import Checkpoint +from thoth.domain.evidence_graph import EvidenceSourceRecord +from thoth.domain.execution_full import PlanExecutionRecord, StepExecutionAttemptRecord +from thoth.domain.hypothesis_full import ( + HypothesisPortfolioRecord, + HypothesisRecord, + HypothesisRelationRecord, +) +from thoth.domain.outcome_full import OutcomeAssessmentRecord, OutcomeSeriesRecord +from thoth.domain.thread_runtime import ThreadCheckpoint +from thoth.ports.model import ModelPort + +STAMP = datetime(2026, 10, 23, 12, tzinfo=UTC) +COMMON: dict[str, object] = { + "project_id": "project:f03b", + "object_id": "object:f03b", + "source_object_id": "object:f03b", + "thread_id": "thread:f03b", + "operation_id": "operation:f03b", + "cycle_id": "cycle:f03b", + "portfolio_id": "portfolio:f03b", + "plan_id": "plan:f03b", + "action_id": "action:f03b", + "authorization_id": "authorization:f03b", + "plan_execution_id": "execution:f03b", + "attempt_id": "attempt:f03b", + "step_id": "step:f03b", + "outcome_series_id": "series:f03b", + "outcome_assessment_id": "assessment:f03b", + "criterion_id": "criterion:f03b", + "hypothesis_id": "hypothesis:f03b", + "source_hypothesis_id": "hypothesis:f03b", + "target_hypothesis_id": "hypothesis:other", + "relation_id": "relation:f03b", + "attention_id": "attention:f03b", + "artifact_id": "artifact:f03b", + "record_id": "record:f03b", + "actor_or_agent_ref": "actor:synthetic", + "created_at": STAMP, + "updated_at": STAMP, + "valid_from": STAMP, + "expires_at": STAMP + timedelta(days=1), + "revision_digest": "1" * 64, + "checkpoint_digest": "1" * 64, + "head_set_digest": "1" * 64, + "resume_token": "1" * 64, + "plan_revision_digest": "1" * 64, + "action_plan_revision_digest": "1" * 64, + "expected_working_head_digest": "1" * 64, + "exact_scope_digest": "1" * 64, + "exact_invocation_digest": "1" * 64, + "environment_digest": "1" * 64, + "comparison_baseline_set_digest": "1" * 64, + "baseline_set_digest": "1" * 64, + "record_digest": "1" * 64, + "source_digest": "1" * 64, + "sha256": "1" * 64, + "revision": 1, + "profile_ref": "profile:synthetic", + "evidence_refs": (), + "required_roles": (), + "payload": {"revision": "A"}, +} + + +@dataclass(frozen=True) +class StoreCase: + table: str + model: type[DomainModel] + store: Callable[[Engine], Any] + append: str + revision_field: str + read: str + read_fields: tuple[str | None, ...] + extra: dict[str, object] = field(default_factory=dict) + latest_list: str | None = None + list_fields: tuple[str, ...] = ("project_id",) + digest_field: str = "revision_digest" + read_last: bool = False + time_field: str = "created_at" + + +CASES = ( + StoreCase( + "action_records", + ActionRecord, + SqliteActionStore, + "add_action", + "action_revision_id", + "read_action", + ("project_id", "action_id", None), + { + "primary_purpose": None, + "proposal_state": "DRAFT", + "specification": {}, + "expected_observation_or_change": {}, + "effect_vector": {}, + "impact_set": {}, + "risk_tier": "R0", + "required_processes": (), + }, + "list_actions", + ), + StoreCase( + "action_portfolios", + ActionPortfolioRecord, + SqliteActionStore, + "add_portfolio", + "portfolio_revision_id", + "read_portfolio", + ("project_id", "portfolio_id", None), + { + "action_refs": (), + "decision_need": "synthetic", + "mandatory_criteria": (), + "enhancing_criteria": (), + }, + "list_portfolios", + ), + StoreCase( + "action_plans", + ActionPlanRecord, + SqliteActionStore, + "add_plan", + "plan_revision_id", + "read_plan", + ("project_id", "plan_id", None), + { + "selected_action_refs": (), + "steps": (), + "dependency_edges": (), + "cumulative_impact": {}, + "required_process_union": (), + "required_role_union": (), + "auto_executable_frontier": (), + "point_of_no_return_steps": (), + }, + "list_plans", + ), + StoreCase( + "action_authorizations", + AuthorizationEnvelopeRecord, + SqliteActionStore, + "add_authorization", + "authorization_revision_id", + "read_authorization", + ("project_id", "authorization_id"), + {"predecessor_output_digests": (), "target_baseline_digests": (), "policy_version": "1"}, + "list_authorizations", + ("project_id", "plan_id"), + ), + StoreCase( + "plan_executions", + PlanExecutionRecord, + SqliteExecutionStore, + "add_execution", + "execution_revision_id", + "read_execution", + ("project_id", "plan_execution_id"), + {"execution_profile_ref": "synthetic"}, + "list_executions", + time_field="updated_at", + ), + StoreCase( + "step_execution_attempts", + StepExecutionAttemptRecord, + SqliteExecutionStore, + "add_attempt", + "attempt_revision_id", + "read_attempt", + ("project_id", "attempt_id"), + { + "attempt_number": 1, + "state": "DISPATCHED", + "input_digests": (), + "target_digests": (), + "idempotency_key": "synthetic", + "delivery_guarantee": "AT_MOST_ONCE", + }, + "list_attempts", + ("project_id", "plan_execution_id"), + ), + StoreCase( + "checkpoints", + Checkpoint, + SqliteEventStore, + "insert_checkpoint", + "checkpoint_id", + "latest_checkpoint", + ("operation_id",), + digest_field="checkpoint_digest", + ), + StoreCase( + "thread_checkpoints", + ThreadCheckpoint, + SqliteThreadRuntimeStore, + "put_checkpoint", + "checkpoint_id", + "list_checkpoints", + ("project_id", "thread_id"), + digest_field="checkpoint_digest", + read_last=True, + ), + StoreCase( + "outcome_series", + OutcomeSeriesRecord, + SqliteOutcomeStore, + "add_series", + "series_revision_id", + "read_series", + ("project_id", "outcome_series_id", None), + {"assessment_windows": (), "phase_states": {}}, + "list_series", + ), + StoreCase( + "outcome_assessments", + OutcomeAssessmentRecord, + SqliteOutcomeStore, + "add_assessment", + "assessment_revision_id", + "read_assessment", + ("project_id", "outcome_assessment_id", None), + { + "assessment_phase": "POST", + "profile_version": 1, + "actual_observation_refs": (), + "comparator_refs": (), + "assumptions": (), + }, + "list_assessments", + ), + StoreCase( + "decision_objects", + DecisionObjectRecord, + SqliteDecisionObjectStore, + "add_object", + "object_revision_id", + "read_object", + ("project_id", "object_id", None), + { + "purpose_statement": "synthetic", + "problem_frame": "synthetic", + "focus_refs": (), + "materialization_trigger": "SYNTHETIC", + "trigger_evidence_refs": (), + "entry_criteria": {}, + "exit_criteria": {}, + }, + "list_objects", + ), + StoreCase( + "object_relations", + ObjectRelationRecord, + SqliteDecisionObjectStore, + "add_relation", + "relation_revision_id", + "read_relation", + ("relation_id",), + { + "relation_type": "RELATED", + "target_ref": "object:other", + "semantic_role": "synthetic", + "authority_state": "INFORMAL", + }, + "list_relations", + ("project_id", "source_object_id"), + ), + StoreCase( + "object_attention", + ObjectAttentionRecord, + SqliteDecisionObjectStore, + "add_attention", + "attention_revision_id", + "read_attention", + ("attention_id",), + {"attention_type": "REVIEW", "risk": "LOW", "reason": "synthetic", "condition_ref": "c"}, + "list_attention", + ("project_id", "object_id"), + ), + StoreCase( + "criterion_contracts", + CriterionContractRecord, + SqliteCriterionContractStore, + "add_contract", + "criterion_revision_id", + "read_contract", + ("project_id", "criterion_id", None), + { + "identity": {"name": "synthetic"}, + "profile_refs": (), + "construct_outcome_definition": "x", + "verification_spec": {}, + "required_evidence": (), + "field_evidence_map": {}, + "field_authority_and_version": {}, + "governance": {}, + }, + "list_contracts", + ), + StoreCase( + "hypothesis_records", + HypothesisRecord, + SqliteHypothesisStore, + "add_hypothesis", + "hypothesis_revision_id", + "read_hypothesis", + ("project_id", "hypothesis_id", None), + { + "statement": "synthetic", + "observed_problem": "synthetic", + "primary_intent": None, + "evidence_basis": "UNKNOWN", + "scope": {}, + }, + "list_hypotheses", + ), + StoreCase( + "hypothesis_portfolios", + HypothesisPortfolioRecord, + SqliteHypothesisStore, + "add_portfolio", + "portfolio_revision_id", + "read_portfolio", + ("project_id", "portfolio_id", None), + {"hypothesis_refs": (), "unknown_reserve": {}}, + "list_portfolios", + ), + StoreCase( + "hypothesis_relations", + HypothesisRelationRecord, + SqliteHypothesisStore, + "add_relation", + "relation_revision_id", + "read_relation", + ("relation_id",), + {"relation_type": "RELATED"}, + "list_relations", + ("project_id", "portfolio_id"), + ), + StoreCase( + "evidence_sources", + EvidenceSourceRecord, + SqliteEvidenceGraphStore, + "add_source", + "source_id", + "read_source_by_artifact", + ("artifact_id",), + { + "connector_ref": "fixture:direct-store", + "uri": "fixture://f03b", + "artifact_type": "TEXT", + "authority_status": "INFORMAL", + "security_class": "INTERNAL", + "cutoff_eligibility": "ELIGIBLE", + "lineage_root_id": "source:root", + }, + digest_field="source_digest", + ), + # The plan classifies this table as safe because it has a monotonic version. Check both APIs. + StoreCase( + "control_records", + ControlRecord, + SqliteControlRecordStore, + "append", + "control_revision_id", + "read", + ("project_id", "namespace", "record_id"), + {"namespace": "MEMORY", "record_type": "CANDIDATE", "version": 1, "state": "CANDIDATE"}, + "list", + ("project_id", "namespace"), + digest_field="record_digest", + ), +) + + +def revisions(case: StoreCase, offset: int) -> tuple[DomainModel, DomainModel]: + values = {key: value for key, value in COMMON.items() if key in case.model.model_fields} + first = case.model.model_validate({**values, **case.extra, case.revision_field: "z:revision-A"}) + second = first.model_dump(mode="python") + second.update( + { + case.revision_field: "a:revision-B", + case.digest_field: "2" * 64, + case.time_field: STAMP + timedelta(seconds=offset), + } + ) + for name in ("revision", "version"): + if isinstance(second.get(name), int): + second[name] = int(str(second[name])) + 1 + if "supersedes_revision_digest" in second: + second["supersedes_revision_digest"] = getattr(first, case.digest_field) + if "payload" in second: + second["payload"] = {"revision": "B"} + return first, case.model.model_validate(second) + + +def observe(case: StoreCase, store: Any, first: DomainModel) -> dict[str, list[str]]: + args = [None if name is None else getattr(first, name) for name in case.read_fields] + current = getattr(store, case.read)(*args) + if case.read_last: + current = current[-1] if current else None + found = {case.read: [] if current is None else [str(getattr(current, case.revision_field))]} + if case.latest_list: + values = getattr(store, case.latest_list)( + *[getattr(first, key) for key in case.list_fields] + ) + found[case.latest_list] = [str(getattr(value, case.revision_field)) for value in values] + if case.read_fields[-1] is None: + historic = getattr(store, case.read)(*args[:-1], getattr(first, case.digest_field)) + assert historic is not None and historic.model_dump() == first.model_dump() + return found + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.table) +@pytest.mark.parametrize("offset", [-60, 0], ids=["rollback", "tie"]) +def test_latest_is_last_appended_revision(tmp_path: Path, case: StoreCase, offset: int) -> None: + path = tmp_path / "store.sqlite3" + engine = create_engine(f"sqlite+pysqlite:///{path.as_posix()}") + table = metadata.tables[case.table] + table.create(engine) + try: + first, second = revisions(case, offset) + store = case.store(engine) + getattr(store, case.append)(first) + getattr(store, case.append)(second) + with engine.connect() as connection: + persisted = [ + dict(row) + for row in connection.execute( + select(literal_column("rowid"), table).order_by(literal_column("rowid")) + ).mappings() + ] + assert len(persisted) == 2 + assert [datetime.fromisoformat(str(row["created_at"])) for row in persisted] == [ + STAMP, + STAMP + timedelta(seconds=offset), + ] + observed = observe(case, store, first) + finally: + engine.dispose() + reopened = create_engine(f"sqlite+pysqlite:///{path.as_posix()}") + try: + after_reopen = observe(case, case.store(reopened), first) + assert after_reopen == observed + finally: + reopened.dispose() + expected = {name: [str(getattr(second, case.revision_field))] for name in observed} + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/f03b-20261023" + / os.environ.get("THOTH_F03B_PHASE", "direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"{case.table}-{'rollback' if offset else 'tie'}.json").write_text( + json.dumps( + { + "table": case.table, + "offset_seconds": offset, + "persisted": persisted, + "observed": observed, + "expected": expected, + "reopen_equal": True, + "paths": { + name: "PASS" if observed[name] == expected[name] else "FAIL" + for name in observed + }, + }, + indent=2, + default=str, + ) + + "\n", + encoding="utf-8", + ) + assert observed == expected + + +@pytest.mark.parametrize("table_name", ["plan_executions", "outcome_series"]) +@pytest.mark.parametrize( + "mode", ["higher_revision", "counter_tie_rollback", "counter_tie_same_time"] +) +def test_counter_order_precedes_time_then_uses_append_order( + tmp_path: Path, table_name: str, mode: str +) -> None: + case = next(case for case in CASES if case.table == table_name) + offset = 60 if mode == "higher_revision" else -60 if mode == "counter_tie_rollback" else 0 + first, second = revisions(case, offset) + first = case.model.model_validate({**first.model_dump(), "revision": 2}) + second = case.model.model_validate( + {**second.model_dump(), "revision": 1 if mode == "higher_revision" else 2} + ) + engine = create_engine(f"sqlite+pysqlite:///{(tmp_path / 'counter.sqlite3').as_posix()}") + metadata.tables[case.table].create(engine) + try: + store = case.store(engine) + getattr(store, case.append)(first) + getattr(store, case.append)(second) + selected = first if mode == "higher_revision" else second + expected = str(getattr(selected, case.revision_field)) + assert all(found == [expected] for found in observe(case, store, first).values()) + finally: + engine.dispose() + + +class NoModels: + calls = 0 + + def resolve(self, *, provider: str, model: str | None) -> ModelPort: + self.calls += 1 + raise AssertionError("F03b metadata flow must not resolve a model") + + +async def test_pause_uses_latest_append_while_checkpoint_history_keeps_display_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + clock = {"now": STAMP} + + def tick(self: SystemClock) -> datetime: + now = clock["now"] + clock["now"] += timedelta(microseconds=1) + return now + + monkeypatch.setattr(SystemClock, "now", tick) + models = NoModels() + runtime = create_runtime(tmp_path / "workspace", model_resolver=models) + project, thread = "project:checkpoint-clock", "thread:checkpoint-clock" + body: dict[str, object] = {"project_id": project, "thread_id": thread} + + async def call(method: str, key: str, payload: dict[str, object]) -> dict[str, Any]: + return value(await runtime.bus.dispatch(request(method, key, payload))) + + try: + await call( + "project/create", + "project", + {"project_id": project, "name": "Clock ordering", "cutoff_at": "2026-10-01T00:00:00Z"}, + ) + await call( + "thread/start", "thread", {**body, "problem": "Check clock ordering", "scope": {}} + ) + first = (await call("thread/pause", "pause-A", body))["checkpoint"] + clock["now"] = STAMP - timedelta(minutes=1) + await call("thread/resume", "resume-A", body) + second = (await call("thread/pause", "pause-B", body))["checkpoint"] + assert first["created_at"] > second["created_at"] + history = (await call("thread/checkpoint/list", "history", body))["checkpoints"] + records = SqliteThreadRuntimeStore(runtime.ledger.engine).list_checkpoints(project, thread) + expected = [ + record.model_dump(mode="json") + for record in sorted( + records, key=lambda record: (record.created_at.isoformat(), record.checkpoint_id) + ) + ] + assert history == expected + assert (await call("thread/pause", "already-paused", body))["checkpoint"] == second + resumed = await call( + "thread/resume", + "resume-B", + {**body, "expected_checkpoint_digest": second["checkpoint_digest"]}, + ) + assert resumed["execution_state"] == "IDLE" + finally: + runtime.close() + assert models.calls == 0 + + +@pytest.mark.parametrize("mode", ["higher_version", "tie_rollback", "tie_same_time", "tie_forward"]) +def test_control_current_version_ties_and_display_history(tmp_path: Path, mode: str) -> None: + case = next(case for case in CASES if case.table == "control_records") + offset = -60 if mode == "tie_rollback" else 0 if mode == "tie_same_time" else 60 + first, second = revisions(case, offset) + first = ControlRecord.model_validate( + {**first.model_dump(), "version": 3 if mode == "higher_version" else 2} + ) + second = ControlRecord.model_validate({**second.model_dump(), "version": 2}) + engine = create_engine(f"sqlite+pysqlite:///{(tmp_path / 'control.sqlite3').as_posix()}") + metadata.tables[case.table].create(engine) + try: + store = SqliteControlRecordStore(engine) + store.append(first) + store.append(second) + current = store.read(first.project_id, first.namespace, first.record_id) + latest = store.list(first.project_id, first.namespace) + history = store.list(first.project_id, first.namespace, latest_only=False) + expected = first if mode == "higher_version" else second + expected_history = tuple( + sorted( + (first, second), key=lambda record: (record.created_at.isoformat(), record.version) + ) + ) + folder = ( + Path(__file__).resolve().parents[2] + / "outputs/f03b-20261023" + / os.environ.get("THOTH_F03B_PHASE", "direct") + ) + folder.mkdir(parents=True, exist_ok=True) + (folder / f"control-{mode}.json").write_text( + json.dumps( + { + "mode": mode, + "current": None if current is None else current.control_revision_id, + "latest": [record.control_revision_id for record in latest], + "history": [record.control_revision_id for record in history], + "expected_current": expected.control_revision_id, + "expected_history": [record.control_revision_id for record in expected_history], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + assert history == expected_history + assert current == expected + assert latest == (expected,) + finally: + engine.dispose() diff --git a/tests/unit/test_journal_completion.py b/tests/unit/test_journal_completion.py new file mode 100644 index 0000000..d145f5f --- /dev/null +++ b/tests/unit/test_journal_completion.py @@ -0,0 +1,319 @@ +"""Controlled journal state validates repair, rollback and fail-closed deferral.""" + +from collections.abc import Generator +from contextlib import contextmanager, nullcontext +from datetime import UTC, datetime, timedelta +from typing import cast + +import pytest +from pydantic import JsonValue + +from thoth.application.services.operation_journal import OperationJournal +from thoth.domain.enums import OperationState +from thoth.domain.event import Checkpoint, EventRecord +from thoth.domain.operation import OperationRecord +from thoth.domain.resource_scope import ResourceScopeError +from thoth.ports.journal import JournalCompletionUnavailable +from thoth.ports.operation import OperationStorePort +from thoth.ports.resource_scope import ResourceAccessPort +from thoth.ports.runtime import AtomicUnitOfWorkPort +from thoth.protocol.journal_completion import JournalCompletion, operation_result_digest +from thoth.protocol.journal_completion_methods import SUPPLEMENTABLE_METHODS, SUPPORTED_RESULT_FORMS + +BASE = datetime(2026, 10, 19, tzinfo=UTC) +METHOD = "project/metadata/update" + + +def test_measured_methods_and_saved_forms_match_reviewed_fixture() -> None: + import json + from pathlib import Path + + from thoth.protocol.read_queries import READ_QUERY_METHODS + from thoth.protocol.registry import PUBLIC_METHODS + + fixture = json.loads( + (Path(__file__).parents[1] / "fixtures/supplementable_methods.json").read_text( + encoding="utf-8" + ) + ) + assert len(SUPPLEMENTABLE_METHODS) == 20 + assert sorted(SUPPLEMENTABLE_METHODS) == fixture["methods"] + assert set(PUBLIC_METHODS) >= SUPPLEMENTABLE_METHODS + assert not SUPPLEMENTABLE_METHODS.intersection(READ_QUERY_METHODS) + assert not SUPPLEMENTABLE_METHODS.intersection( + { + "thread/start", + "thread/input", + "thread/steer", + "thread/pause", + "thread/resume", + "thread/stop", + "revision/restore", + "revision/restore/apply", + "revision/changeSet/commit", + } + ) + assert { + method: sorted(sorted(keys) for keys in forms) + for method, forms in SUPPORTED_RESULT_FORMS.items() + } == fixture["result_key_shapes"] + assert fixture["evidence"]["path"] == "outputs/refactor-vj-20261019/method-evidence.json" + + +class Clock: + def __init__(self) -> None: + self.value = BASE - timedelta(seconds=1) + self.fixed = False + self.calls = 0 + + def now(self) -> datetime: + value = self.value + self.calls += 1 + if not self.fixed: + self.value += timedelta(microseconds=1) + return value + + +class Ids: + def __init__(self) -> None: + self.calls = 0 + self.tag = "z" + + def new(self, prefix: str) -> str: + self.calls += 1 + return f"{prefix}:{self.tag}-{self.calls:05}" + + +class State: + def __init__(self) -> None: + result: dict[str, JsonValue] = dict.fromkeys(SUPPORTED_RESULT_FORMS[METHOD][0]) + result.update(schema_version="1.0.0", project_id="project:j-unit", name="Saved", revision=1) + self.operation = OperationRecord( + operation_id="operation:j-unit", + project_id="project:j-unit", + method=METHOD, + idempotency_key="key", + scope_digest="a" * 64, + state=OperationState.SUCCEEDED, + result=result, + resource_uses=(), + created_at=BASE - timedelta(seconds=1), + completed_at=BASE, + ) + self.events: list[EventRecord] = [] + self.points: list[Checkpoint] = [] + self.fail_write = False + self.deny = False + self.in_transaction = False + + def read(self, operation_id: str) -> OperationRecord: + assert operation_id == self.operation.operation_id + return self.operation + + def require_operation(self, operation: OperationRecord) -> None: + assert self.in_transaction and operation.operation_id == self.operation.operation_id + if self.deny: + raise ResourceScopeError("RESOURCE_SCOPE_READ_DENIED") + + def list_events(self, operation_id: str) -> tuple[EventRecord, ...]: + return tuple(sorted(self.events, key=lambda e: (e.created_at, e.event_id))) + + def list_checkpoints(self, operation_id: str) -> tuple[Checkpoint, ...]: + return tuple(self.points) + + def latest_checkpoint(self, operation_id: str) -> Checkpoint | None: + return self.points[-1] if self.points else None + + def insert_event(self, event: EventRecord) -> None: + self.events.append(event) + if self.fail_write: + raise JournalCompletionUnavailable("synthetic write failure") + + def insert_checkpoint(self, checkpoint: Checkpoint) -> None: + self.points.append(checkpoint) + + @contextmanager + def transaction(self) -> Generator[None]: + before = self.events[:], self.points[:] + self.in_transaction = True + try: + yield + except BaseException: + self.events[:], self.points[:] = before + raise + finally: + self.in_transaction = False + + @contextmanager + def completion_transaction(self, uow: AtomicUnitOfWorkPort) -> Generator[None]: + with uow.transaction(): + yield + + +def setup() -> tuple[State, Clock, Ids, OperationJournal, JournalCompletion]: + state, clock, ids = State(), Clock(), Ids() + journal = OperationJournal(state, clock, ids) + journal.append( + project_id=state.operation.project_id, + operation_id=state.operation.operation_id, + event_type="operation.started", + payload={"method": METHOD}, + ) + clock.value = BASE + timedelta(seconds=10) + completion = JournalCompletion( + workspace_key="workspace:unit", + pin_workspace=lambda: nullcontext(True), + operations=cast(OperationStorePort, state), + journal=journal, + store=state, + uow=state, + resource_access=cast(ResourceAccessPort, state), + ) + return state, clock, ids, journal, completion + + +def test_repair_is_marked_and_second_call_allocates_and_writes_nothing() -> None: + state, clock, ids, _, completion = setup() + original = state.operation.model_dump_json() + outcome = completion.attempt(state.operation) + assert outcome.status == "SUPPLEMENTED" and outcome.added == 3 + assert state.operation.model_dump_json() == original + assert [e.event_type for e in state.events] == [ + "operation.started", + "project/metadata/updated", + "operation.succeeded", + ] + for item in [*state.events[1:], *state.points]: + assert item.payload["supplemented"] is True + assert item.payload["supplemented_for"] == BASE.isoformat() + assert item.created_at >= BASE + timedelta(seconds=10) + before = state.events[:], state.points[:], clock.calls, ids.calls + assert completion.attempt(state.operation).status == "COMPLETE" + assert (state.events, state.points, clock.calls, ids.calls) == before + + +@pytest.mark.parametrize("marked", [False, True]) +def test_partial_original_or_supplemented_tail_keeps_existing_bytes(marked: bool) -> None: + state, _, _, journal, completion = setup() + assert state.operation.result is not None + digest = operation_result_digest(state.operation.result) + mark: dict[str, JsonValue] = ( + {"supplemented": True, "supplemented_for": BASE.isoformat()} if marked else {} + ) + journal.checkpoint( + operation_id=state.operation.operation_id, + payload={"state": "SUCCEEDED", "result_digest": digest, **mark}, + ) + journal.append( + project_id=state.operation.project_id, + operation_id=state.operation.operation_id, + event_type="project/metadata/updated", + payload={"method": METHOD, "result_digest": digest, **mark}, + ) + before = ( + [e.model_dump_json() for e in state.events], + [p.model_dump_json() for p in state.points], + ) + outcome = completion.attempt(state.operation) + assert outcome.status == "SUPPLEMENTED" and outcome.added == 1 + assert [e.model_dump_json() for e in state.events[:2]] == before[0] + assert [p.model_dump_json() for p in state.points] == before[1] + + +@pytest.mark.parametrize( + "case", + [ + "unknown-method", + "failed", + "missing-result", + "missing-completed-at", + "unknown-shape", + "unknown-schema", + ], +) +def test_unsupported_saved_records_are_not_repaired(case: str) -> None: + state, clock, ids, _, completion = setup() + changes: dict[str, object] = {} + if case == "unknown-method": + changes["method"] = "revision/changeSet/commit" + if case == "failed": + changes.update( + state=OperationState.FAILED, result=None, error={"reason_code": "OPERATION_OWNER_GONE"} + ) + if case == "missing-result": + changes["result"] = None + if case == "missing-completed-at": + changes["completed_at"] = None + if case == "unknown-shape": + changes["result"] = {"unknown": True} + if case == "unknown-schema": + changes["schema_version"] = "2.0.0" + state.operation = state.operation.model_copy(update=changes) + before = clock.calls, ids.calls, state.events[:], state.points[:] + assert completion.attempt(state.operation).status in {"DEFERRED", "NOT_ELIGIBLE"} + assert (clock.calls, ids.calls, state.events, state.points) == before + + +@pytest.mark.parametrize( + "case", ["chain", "duplicate", "payload", "publisher", "checkpoints", "checkpoint-payload"] +) +def test_ambiguous_or_broken_existing_journal_defers_without_writes(case: str) -> None: + state, clock, ids, journal, completion = setup() + assert state.operation.result is not None + digest = operation_result_digest(state.operation.result) + if case == "chain": + state.events[0] = state.events[0].model_copy(update={"event_digest": "f" * 64}) + if case in {"duplicate", "payload", "publisher"}: + for _ in range(2 if case == "duplicate" else 1): + journal.append( + project_id=state.operation.project_id, + operation_id=state.operation.operation_id, + event_type="research.terminal" + if case == "publisher" + else "project/metadata/updated", + payload={ + "method": METHOD, + "result_digest": "f" * 64 if case == "payload" else digest, + }, + ) + if case in {"checkpoints", "checkpoint-payload"}: + for _ in range(2 if case == "checkpoints" else 1): + journal.checkpoint( + operation_id=state.operation.operation_id, + payload={ + "state": "SUCCEEDED", + "result_digest": "f" * 64 if case == "checkpoint-payload" else digest, + }, + ) + before = state.events[:], state.points[:], clock.calls, ids.calls + assert completion.attempt(state.operation).status == "DEFERRED" + assert (state.events, state.points, clock.calls, ids.calls) == before + + +@pytest.mark.parametrize("case", ["rollback-clock", "equal-time-reversed-id", "storage"]) +def test_new_batch_is_rolled_back_on_bad_order_or_storage_failure(case: str) -> None: + state, clock, ids, _, completion = setup() + if case == "rollback-clock": + clock.value = BASE - timedelta(seconds=5) + if case == "equal-time-reversed-id": + clock.value, clock.fixed, ids.tag = state.events[0].created_at, True, "a" + if case == "storage": + state.fail_write = True + before = state.events[:], state.points[:], state.operation.model_dump_json() + outcome = completion.attempt(state.operation) + assert outcome.status == "DEFERRED" + assert (state.events, state.points, state.operation.model_dump_json()) == before + + +@pytest.mark.parametrize("case", ["denied", "canonical-change"]) +def test_authority_and_canonical_failures_are_not_swallowed(case: str) -> None: + state, _, _, _, completion = setup() + original = state.operation + if case == "denied": + state.deny = True + else: + state.operation = original.model_copy(update={"scope_digest": "c" * 64}) + before = state.events[:], state.points[:] + with pytest.raises(ResourceScopeError): + completion.attempt(original) + assert (state.events, state.points) == before diff --git a/tests/unit/test_journal_completion_gate.py b/tests/unit/test_journal_completion_gate.py new file mode 100644 index 0000000..2dc4a04 --- /dev/null +++ b/tests/unit/test_journal_completion_gate.py @@ -0,0 +1,92 @@ +"""The journal gate is reentrant, shared and leak-free; result digest bytes are frozen.""" + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Event + +import pytest +from pydantic import JsonValue + +from thoth.domain.errors import CanonicalizationError +from thoth.protocol import journal_completion_gate as gates +from thoth.protocol.journal_completion import operation_result_digest + +VECTORS = json.loads( + (Path(__file__).parents[1] / "fixtures/operation_result_digest_vectors.json").read_text( + encoding="utf-8" + ) +)["vectors"] + + +@pytest.mark.parametrize("vector", VECTORS) +def test_digest_bytes_match_cp56_vectors(vector: dict[str, object]) -> None: + from typing import cast + + actual = operation_result_digest(cast(dict[str, JsonValue], vector["value"])) + assert actual.encode("ascii") == str(vector["digest"]).encode("ascii") + + +def test_binary_float_still_fails_canonicalization() -> None: + with pytest.raises(CanonicalizationError, match="binary float"): + operation_result_digest({"value": 0.25}) + + +def test_reentrant_gate_releases_after_exception() -> None: + key = ("workspace:gate-unit", "operation:gate-unit") + entries = vars(gates)["_entries"] + assert key not in entries + with pytest.raises(RuntimeError, match="injected"), gates.journal_gate(*key): + entry = entries[key] + assert entry.users == 1 + with gates.journal_gate(*key): + assert entries[key] is entry and entry.users == 2 + assert entry.users == 1 + raise RuntimeError("injected") + assert key not in entries + + +def test_two_threads_share_gate_and_remove_last_reference() -> None: + key = ("workspace:threads", "operation:threads") + first_entered, second_attempted, release_first, second_entered = (Event() for _ in range(4)) + order: list[str] = [] + + def first() -> None: + with gates.journal_gate(*key): + first_entered.set() + assert release_first.wait(timeout=5) + order.append("first") + + def second() -> None: + second_attempted.set() + with gates.journal_gate(*key): + second_entered.set() + order.append("second") + + with ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(first) + assert first_entered.wait(timeout=5) + b = pool.submit(second) + try: + assert second_attempted.wait(timeout=5) + assert not second_entered.wait(timeout=0.05) + finally: + release_first.set() + a.result(timeout=5) + b.result(timeout=5) + assert order == ["first", "second"] + assert key not in vars(gates)["_entries"] + + +def test_same_operation_in_other_workspace_uses_a_different_gate() -> None: + with ( + gates.journal_gate("workspace:a", "operation:same"), + ThreadPoolExecutor(max_workers=1) as pool, + ): + + def other() -> str: + with gates.journal_gate("workspace:b", "operation:same"): + return "entered" + + assert pool.submit(other).result(timeout=5) == "entered" + assert not {key for key in vars(gates)["_entries"] if key[1] == "operation:same"}