From 9c79082832cd0c22892dcf465f9d4b7b63979d59 Mon Sep 17 00:00:00 2001 From: THOTH Date: Fri, 23 Oct 2026 17:55:12 +0900 Subject: [PATCH 1/3] Run the full Python suite in CI and turn hypothesis contract v3 on by default Full Python suite in CI - The PR check ran no Python tests; the full suite (over 5,500 tests, more than four hours on one machine) was run locally before each merge. CI now runs it on every pull request, on main and nightly (18:00 UTC), in nine parallel Windows jobs: - python-fast: every test outside tests/integration. - python-integration: tests/integration split into eight shards by measured file duration. The shard script fails the job unless the shards cover every integration test file exactly once; new files without a measured time go to the lightest shard. - python-summary: fails on a missing, empty, damaged or duplicated JUnit report, any test failure or error, or a failed, cancelled or skipped job. - Same pytest options and test selection as the local full run. No test, expectation or time limit changed. Manual dispatch is also available. Hypothesis contract v3 on by default - Projects with no saved choice now use the v3 contract (expected result per hypothesis for each test, and refutation conditions). A saved False keeps the v2 contract, and saved settings are not rewritten. Live comparisons on two synthetic datasets passed the fixed acceptance criteria in all recent runs; predictions remain labelled as not confirmed by evidence. README and KNOWN_LIMITATIONS describe the default and the extra model call. Verification: shard and summary tool tests (24); collected node IDs across shards match the full collection (5,613 tests, no overlap); hypothesis contract, settings and research request tests and web switch/settings tests pass; actionlint, ruff, types, lint, budget and manifest checks pass. This PR's CI run is the first full-suite run of the new jobs. --- .github/workflows/product-check.yml | 129 ++++++- README.ko.md | 5 + README.md | 5 + SOURCE_MANIFEST.json | 79 ++-- .../HypothesisContractSwitch.test.tsx | 35 +- .../components/HypothesisContractSwitch.tsx | 2 +- apps/web/src/i18n/modelHelpMessages.ts | 4 +- docs/KNOWN_LIMITATIONS.ko.md | 7 + docs/KNOWN_LIMITATIONS.md | 8 + docs/architecture/verification-profiles.md | 21 + scripts/ci_test_shards.py | 177 +++++++++ scripts/ci_test_summary.py | 71 ++++ .../services/research_attempt_runner.py | 2 +- src/thoth/domain/model_call_settings.py | 6 +- tests/ci/integration_durations.json | 363 ++++++++++++++++++ tests/ci/test_ci_test_shards.py | 137 +++++++ tests/ci/test_ci_test_summary.py | 56 +++ .../test_hypothesis_contract_v3_run.py | 53 ++- tests/integration/test_model_call_settings.py | 2 +- 19 files changed, 1113 insertions(+), 49 deletions(-) create mode 100644 scripts/ci_test_shards.py create mode 100644 scripts/ci_test_summary.py create mode 100644 tests/ci/integration_durations.json create mode 100644 tests/ci/test_ci_test_shards.py create mode 100644 tests/ci/test_ci_test_summary.py diff --git a/.github/workflows/product-check.yml b/.github/workflows/product-check.yml index 0fd2bcb..ee4864c 100644 --- a/.github/workflows/product-check.yml +++ b/.github/workflows/product-check.yml @@ -4,6 +4,10 @@ on: pull_request: push: branches: [main] + schedule: + # 18:00 UTC is 03:00 KST the following day; scheduled runs use the default branch. + - cron: "0 18 * * *" + workflow_dispatch: permissions: contents: read @@ -15,8 +19,7 @@ env: jobs: checks: - # Fast public checks (about ten minutes). The full Python suite takes hours on a hosted - # runner, so maintainers run it locally before merging. + # Public static, Web and quick-start checks. The jobs below cover the full Python suite. runs-on: windows-latest timeout-minutes: 60 defaults: @@ -85,3 +88,125 @@ jobs: } finally { & taskkill /PID $thothApi.Id /T /F | Out-Null } + + python-fast: + runs-on: windows-latest + timeout-minutes: 60 + defaults: + run: + shell: pwsh + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.thoth-test-browsers + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: actions/setup-node@v4 + with: + node-version: "24" + - uses: pnpm/action-setup@v4 + with: + version: "11.21.0" + - uses: astral-sh/setup-uv@v7 + with: + version: "0.12.5" + - name: Install locked test dependencies + run: | + python -m venv --copies --without-pip .venv + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv sync --python .\.venv\Scripts\python.exe --no-managed-python --no-python-downloads --frozen --extra dev --dev --extra browser + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + pnpm install --frozen-lockfile + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + .\.venv\Scripts\python.exe -m playwright install --only-shell chromium + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Non-integration Python tests + run: .\.venv\Scripts\python.exe -m pytest --ignore=tests/integration -q -rfE -p no:cacheprovider --junitxml=test-results/python-fast.xml + - name: Upload Python test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: python-junit-fast + path: test-results/python-fast.xml + if-no-files-found: error + + python-integration: + runs-on: windows-latest + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7] + defaults: + run: + shell: pwsh + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.thoth-test-browsers + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: actions/setup-node@v4 + with: + node-version: "24" + - uses: pnpm/action-setup@v4 + with: + version: "11.21.0" + - uses: astral-sh/setup-uv@v7 + with: + version: "0.12.5" + - name: Install locked test dependencies + run: | + python -m venv --copies --without-pip .venv + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv sync --python .\.venv\Scripts\python.exe --no-managed-python --no-python-downloads --frozen --extra dev --dev --extra browser + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + pnpm install --frozen-lockfile + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + .\.venv\Scripts\python.exe -m playwright install --only-shell chromium + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Validate and run integration shard + run: | + $thothFilesJson = & .\.venv\Scripts\python.exe scripts/ci_test_shards.py select --count 8 --index ${{ matrix.shard }} + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $thothFiles = @($thothFilesJson | ConvertFrom-Json) + if ($thothFiles.Count -eq 0) { throw 'Integration shard is empty.' } + & .\.venv\Scripts\python.exe -m pytest @thothFiles -q -rfE -p no:cacheprovider --junitxml=test-results/python-integration-${{ matrix.shard }}.xml + exit $LASTEXITCODE + - name: Upload Python test results + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: python-junit-integration-${{ matrix.shard }} + path: test-results/python-integration-${{ matrix.shard }}.xml + if-no-files-found: error + + python-summary: + needs: [python-fast, python-integration] + if: ${{ always() }} + runs-on: windows-latest + timeout-minutes: 10 + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: actions/download-artifact@v8 + with: + pattern: python-junit-* + path: test-results/aggregate + merge-multiple: true + - name: Summarize every Python group + if: ${{ always() }} + env: + FAST_RESULT: ${{ needs.python-fast.result }} + INTEGRATION_RESULT: ${{ needs.python-integration.result }} + run: >- + python scripts/ci_test_summary.py --reports test-results/aggregate --shards 8 + --job-results "$env:FAST_RESULT" "$env:INTEGRATION_RESULT" + --markdown-output "$env:GITHUB_STEP_SUMMARY" diff --git a/README.ko.md b/README.ko.md index 08394c5..1afff4f 100644 --- a/README.ko.md +++ b/README.ko.md @@ -91,6 +91,11 @@ ## 다시 살펴볼 수 있는 연구 흐름 +가설별 예상 결과와 기각 조건은 저장된 선택이 없는 프로젝트에서 기본으로 켜집니다. +이전에 꺼서 저장한 설정은 그대로 유지되며, 프로젝트의 고급 설정에서 바꿀 수 있습니다. +빈 예측을 채우는 추가 모델 호출과 토큰 사용이 생길 수 있습니다. 근거를 연결한 예측도 +확인 전에는 미확정 제안입니다. + | 단계 | THOTH에서 하는 일 | | :--- | :--- | | **1 · 질문** | 연구 질문을 만들고 관련 문서를 연결합니다. | diff --git a/README.md b/README.md index 124c6c1..53e52df 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,11 @@ Older requests retain their original language policy. New questions use model ca ## A research loop you can revisit +Expected results and refutation conditions are on by default for projects without a saved +choice. A previously saved off choice stays off. You can change this in the project's advanced +settings. Filling missing predictions may add a model call and use more tokens; predictions +remain unconfirmed proposals, even when they include evidence references. + | Step | In THOTH | | :--- | :--- | | **1 · Ask** | Start with a research question and connect the relevant documents. | diff --git a/SOURCE_MANIFEST.json b/SOURCE_MANIFEST.json index d4fe4d9..6de503b 100644 --- a/SOURCE_MANIFEST.json +++ b/SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schema_version": "thoth-source-preview-manifest-v1", - "created_at": "2026-10-23T04:39:51.014673+00:00", + "created_at": "2026-10-23T08:49:28.623906+00:00", "license": "MIT", "git_history_included": false, "full_suite_verified": false, @@ -18,8 +18,8 @@ }, { "path": ".github/workflows/product-check.yml", - "sha256": "6e58032f4e10fc0c7e896ab6ddc1730021aececb440be781ffe9d7159ae77ca9", - "bytes": 3386 + "sha256": "418474e27a6faf3dda070566a987a69a9b8b5f995f94ca83e0d376012cd0c717", + "bytes": 8122 }, { "path": ".gitignore", @@ -53,13 +53,13 @@ }, { "path": "README.ko.md", - "sha256": "8e615dcc545c0086d81f84d90f4305bc8a7e7df9475307515bc68c7b65e57602", - "bytes": 15926 + "sha256": "6398580db8d526eb995377d437f1703fbf0dc250f430ab167764771be7e7a5f8", + "bytes": 16330 }, { "path": "README.md", - "sha256": "43f2fd9cebc4acd2278ab5ee57c88005714727c9eb73f6a5327ef46c86a3bcc9", - "bytes": 13723 + "sha256": "2617a0c27e193ed5e190edb604c97954d7e7a658962dd5c44e51eef8aa4b9a4e", + "bytes": 14075 }, { "path": "THIRD_PARTY_NOTICES.md", @@ -728,13 +728,13 @@ }, { "path": "apps/web/src/components/HypothesisContractSwitch.test.tsx", - "sha256": "3cecd4c579375e933a54dd1644a4724f87efd951b23091581fe178e0ae945f47", - "bytes": 3167 + "sha256": "8f59233c433408a935bce18447b3cf0358c2843be5662e193e2886df5d6ef6a3", + "bytes": 4119 }, { "path": "apps/web/src/components/HypothesisContractSwitch.tsx", - "sha256": "47601c363d7d2a832f574b2c8587a2601b3439b68ae64cc02bf7ac2181dad067", - "bytes": 1753 + "sha256": "92f2b19cf0d8684efbd57019d1a6c4622878fef40783f2763309b8e334417c8b", + "bytes": 1703 }, { "path": "apps/web/src/components/HypothesisFields.tsx", @@ -1653,8 +1653,8 @@ }, { "path": "apps/web/src/i18n/modelHelpMessages.ts", - "sha256": "a674ce712e9efcc052851c487946d7872973328d3e46fc7d00c93099ac23a1f9", - "bytes": 2392 + "sha256": "fde35f0cc7f76a9dd59f5539b695deda068a823f5c42e064da9e345bf6231cb7", + "bytes": 2530 }, { "path": "apps/web/src/i18n/oauthMessages.ts", @@ -1973,13 +1973,13 @@ }, { "path": "docs/KNOWN_LIMITATIONS.ko.md", - "sha256": "6304fc90fc010d75c995fb693170c9060a0d13c327cb0c540e1d6b2594e29dca", - "bytes": 12832 + "sha256": "60a119b205924532f97dc3b8137d1c8248e8dcd8728193a47d7a04c052d01f81", + "bytes": 13386 }, { "path": "docs/KNOWN_LIMITATIONS.md", - "sha256": "fbd264eba5ee88f650d85f04eade4a21ca6b4d45e790d028a1ad7163b8bafe9f", - "bytes": 10605 + "sha256": "2be5949bcbbb3a5ea54e5f0b29688fec7de15f8439737409711107766e87deff", + "bytes": 11117 }, { "path": "docs/KNOWN_TEST_FAILURES.json", @@ -2108,8 +2108,8 @@ }, { "path": "docs/architecture/verification-profiles.md", - "sha256": "05d1c7d5d776b23f27536c711dee5fdbaf89fec0472afccee47e6c5263f5bf34", - "bytes": 2656 + "sha256": "edbd7848300938086f040249f2aa88c811506509cde55cc1f5508a3b0628f466", + "bytes": 4307 }, { "path": "docs/brand/README.md", @@ -3056,6 +3056,16 @@ "sha256": "bdaa3aeff1c58354a44d36011eca256cdd08f01a534e361b8d86c0b2fce35de2", "bytes": 7739 }, + { + "path": "scripts/ci_test_shards.py", + "sha256": "dd47bf53bc54e90e5dc91d192d220402376955326f88f394232118eb6f8c7a3c", + "bytes": 7452 + }, + { + "path": "scripts/ci_test_summary.py", + "sha256": "408c84601430e119f2a0ad2d079903b9d35ba5b79a42e75e4c266d7bd0f2f466", + "bytes": 2950 + }, { "path": "scripts/complete_architecture_gate.py", "sha256": "820a993afcdae5fe1829a4edd326264ac7b913849ea4c518835292c0436181a3", @@ -5578,8 +5588,8 @@ }, { "path": "src/thoth/application/services/research_attempt_runner.py", - "sha256": "5e6499afabe6ea7d2d10f80123e8070f55eee4a265c02d2c06901e0d329dc088", - "bytes": 18958 + "sha256": "db8a20ec7514b466fe289ba41edba517a6ba45e448161cc7c3bbd9588ced9236", + "bytes": 18959 }, { "path": "src/thoth/application/services/research_basis_capture.py", @@ -6698,8 +6708,8 @@ }, { "path": "src/thoth/domain/model_call_settings.py", - "sha256": "ce35097577e69f060963892e485975758125b07a017ff85a881fc59836b7d9e5", - "bytes": 1071 + "sha256": "a498edf0dd8a75861a859ac31ac4b71cd1d3665b725a75ffbfdd6963df7b4188", + "bytes": 1068 }, { "path": "src/thoth/domain/model_catalog.py", @@ -7796,6 +7806,21 @@ "sha256": "eb1bdc5459f0d99b4dbad8ca8fe9d2f0274837a3ffd00f21da7a7a2c04f8c258", "bytes": 4708 }, + { + "path": "tests/ci/integration_durations.json", + "sha256": "4863d6c2f6572bd3aa70dfbc16d50c6a17cc1a223a1343b5af8f20ae7d956473", + "bytes": 23221 + }, + { + "path": "tests/ci/test_ci_test_shards.py", + "sha256": "8233774edbf652f283e837d1d6bc816d3895a2ff40256f7fa1983ee55fba0970", + "bytes": 5139 + }, + { + "path": "tests/ci/test_ci_test_summary.py", + "sha256": "ba729abb8d3dc01f52853ca118e7665c58985790329b4ea16c488b991aea6fda", + "bytes": 2561 + }, { "path": "tests/cli_contract_capture.py", "sha256": "e35e3ce9e618b79fbc8c3b1d1be535fee55ecce344c9266b0e3728860e65f540", @@ -8813,8 +8838,8 @@ }, { "path": "tests/integration/test_hypothesis_contract_v3_run.py", - "sha256": "b8d2cdd96502289242aab2ab9cba73cfd70d61e5c76fd377cde681affc4db2a1", - "bytes": 17533 + "sha256": "ba58f7b95b9936e9077b8f6d67115eb793d7259f8130ccf9ee7fb125c42852b4", + "bytes": 19143 }, { "path": "tests/integration/test_hypothesis_full_rpc.py", @@ -9123,8 +9148,8 @@ }, { "path": "tests/integration/test_model_call_settings.py", - "sha256": "f1fec431b8a9b0ab7e7c370e30a39ccbe2f0ed0ca61c779d34f79734cf2a4020", - "bytes": 8862 + "sha256": "b4f5c6f2ecfab8c9c63a0fd0b05ef248edc9312206b5dfcf9eb5f3c64c3600f3", + "bytes": 8861 }, { "path": "tests/integration/test_model_catalog_refresh_rpc.py", diff --git a/apps/web/src/components/HypothesisContractSwitch.test.tsx b/apps/web/src/components/HypothesisContractSwitch.test.tsx index 116cd1c..b5cac5b 100644 --- a/apps/web/src/components/HypothesisContractSwitch.test.tsx +++ b/apps/web/src/components/HypothesisContractSwitch.test.tsx @@ -4,18 +4,19 @@ import { act } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { HypothesisContractSwitch } from "./HypothesisContractSwitch"; +import { setLocale } from "../i18n/locale"; -const state = vi.hoisted(() => ({ calls: [] as { method: string; input: Record }[], on: false, fail: false })); +const state = vi.hoisted(() => ({ calls: [] as { method: string; input: Record }[], on: true, fail: false, digest: null as string | null })); vi.mock("../api/rpcClient", async importOriginal => ({ ...(await importOriginal()), rpc: async (method: string, input: Record) => { state.calls.push({ method, input }); - if (method === "model/callSettings/update") { if (state.fail) throw new Error("no"); state.on = input.hypothesis_contract_v3 === true; } - return { value: { auto_retry_interrupted_model_call: false, hypothesis_contract_v3: state.on, settings_digest: "d".repeat(64) }, state: "SUCCEEDED", operation_id: "op" }; + if (method === "model/callSettings/update") { if (state.fail) throw new Error("no"); state.on = input.hypothesis_contract_v3 === true; state.digest = "d".repeat(64); } + return { value: { auto_retry_interrupted_model_call: false, hypothesis_contract_v3: state.on, settings_digest: state.digest }, state: "SUCCEEDED", operation_id: "op" }; } })); let root: Root | undefined; let node: HTMLDivElement; -beforeEach(() => { state.calls = []; state.on = false; state.fail = false; }); -afterEach(async () => { if (root) await act(async () => root!.unmount()); root = undefined; node?.remove(); }); +beforeEach(() => { state.calls = []; state.on = true; state.fail = false; state.digest = null; }); +afterEach(async () => { if (root) await act(async () => root!.unmount()); root = undefined; node?.remove(); setLocale("ko"); }); const settle = async () => { for (let i = 0; i < 6; i++) await act(async () => { await new Promise(done => setTimeout(done, 5)); }); }; async function mount() { Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); @@ -26,16 +27,33 @@ async function mount() { } const input = () => node.querySelector("input[type=checkbox]")!; -it("shows the switch off, says in easy words what it does and that it is off unless turned on", async () => { +it("shows the unsaved server default on and explains saved choices and usage", async () => { await mount(); - expect(input().checked).toBe(false); + expect(input().checked).toBe(true); expect(node.textContent).toContain("가설 만들 때 예상 결과와 기각 조건도 함께 받기"); - expect(node.textContent).toContain("기본은 꺼짐"); + expect(node.textContent).toContain("저장된 선택이 없으면 기본으로 켜집니다"); + expect(node.textContent).toContain("이전에 끈 설정은 그대로 유지됩니다"); expect(node.textContent).toContain("추가 모델 호출이 생길 수 있습니다"); expect(node.textContent).not.toMatch(/v3|hypothesis_portfolio|expected_by_hypothesis/); }); +it("keeps a saved off choice visible without writing a new default", async () => { + state.on = false; state.digest = "d".repeat(64); + await mount(); + expect(input().checked).toBe(false); + expect(state.calls.some(call => call.method === "model/callSettings/update")).toBe(false); +}); + +it("explains the same default and saved-off policy in English", async () => { + setLocale("en"); + await mount(); + expect(input().checked).toBe(true); + expect(node.textContent).toContain("On by default when no choice has been saved"); + expect(node.textContent).toContain("Saved off choices stay off"); +}); + it("sends only its own switch and the digest it read, and shows what the server says afterwards", async () => { + state.on = false; state.digest = "d".repeat(64); await mount(); await act(async () => input().click()); await settle(); @@ -45,6 +63,7 @@ it("sends only its own switch and the digest it read, and shows what the server }); it("says so when the setting could not be changed", async () => { + state.on = false; state.digest = "d".repeat(64); state.fail = true; await mount(); await act(async () => input().click()); diff --git a/apps/web/src/components/HypothesisContractSwitch.tsx b/apps/web/src/components/HypothesisContractSwitch.tsx index 0d405ee..1b6c084 100644 --- a/apps/web/src/components/HypothesisContractSwitch.tsx +++ b/apps/web/src/components/HypothesisContractSwitch.tsx @@ -6,7 +6,7 @@ import { rpc } from "../api/rpcClient"; type CallSettings = { auto_retry_interrupted_model_call: boolean; hypothesis_contract_v3: boolean; settings_digest: string | null }; -/** A project's switch for asking the generator for expected results and refutation conditions (off unless turned on). It shows what the server says. */ +/** Show the server's effective expected-results choice, including its default and saved overrides. */ export function HypothesisContractSwitch({ projectId }: { projectId: string }) { const client = useQueryClient(); const key = ["model-call-settings", projectId]; diff --git a/apps/web/src/i18n/modelHelpMessages.ts b/apps/web/src/i18n/modelHelpMessages.ts index cf82367..25ac35f 100644 --- a/apps/web/src/i18n/modelHelpMessages.ts +++ b/apps/web/src/i18n/modelHelpMessages.ts @@ -16,8 +16,8 @@ export const modelHelpMessages = { "en": "The setting could not be changed. The view was refreshed; try again." }, "modelHelp.hypothesis": { - "ko": "켜면 가설을 만드는 호출에서 각 시험의 가설별 예상 결과와 기각 조건을 함께 받아 시험 순서를 정합니다. 비어 있는 예상 결과를 채우기 위해 추가 모델 호출이 생길 수 있습니다. 답이 길어져 토큰이 더 쓰일 수 있으며, 모델이 낸 내용은 미확정 제안입니다. 기본은 꺼짐입니다.", - "en": "Collects each test's expected results for each hypothesis and refutation conditions during hypothesis generation. Filling missing expected results may require an additional model call. Longer answers can use more tokens. Model output remains an unconfirmed proposal. Off by default." + "ko": "켜면 가설을 만드는 호출에서 각 시험의 가설별 예상 결과와 기각 조건을 함께 받아 시험 순서를 정합니다. 비어 있는 예상 결과를 채우기 위해 추가 모델 호출이 생길 수 있습니다. 답이 길어져 토큰이 더 쓰일 수 있으며, 모델이 낸 내용은 미확정 제안입니다. 저장된 선택이 없으면 기본으로 켜집니다. 이전에 끈 설정은 그대로 유지됩니다.", + "en": "Collects each test's expected results for each hypothesis and refutation conditions during hypothesis generation. Filling missing expected results may require an additional model call. Longer answers can use more tokens. Model output remains an unconfirmed proposal. On by default when no choice has been saved. Saved off choices stay off." }, "modelHelp.hypothesisReadFailed": { "ko": "가설 생성 설정을 읽지 못했습니다.", diff --git a/docs/KNOWN_LIMITATIONS.ko.md b/docs/KNOWN_LIMITATIONS.ko.md index 5e702b6..ebbc33e 100644 --- a/docs/KNOWN_LIMITATIONS.ko.md +++ b/docs/KNOWN_LIMITATIONS.ko.md @@ -7,6 +7,13 @@ THOTH는 실험 단계의 소스 미리보기입니다. 이 문서는 저장된 English: [KNOWN_LIMITATIONS.md](KNOWN_LIMITATIONS.md) +## 예상 결과는 검증된 관측이 아닙니다 + +예상 결과 표와 기각 조건은 저장된 선택이 없을 때만 기본으로 켜집니다. 저장된 false는 +기록을 다시 쓰지 않고 유지합니다. 빈칸 보충에는 추가 모델 호출이 생길 수 있고 출력이 +길어지면 토큰을 더 씁니다. 예측은 관측이 아니며, 근거 연결만으로 인과 주장이 검증되지는 +않습니다. 근거 없는 행은 미확정 예측입니다. 기본값 변경으로 기존 결과를 다시 생성하지 않습니다. + ## 로컬 모드는 내 컴퓨터의 사용자를 신뢰합니다 로컬 모드는 한 사람이 사용하는 환경을 전제로, 같은 컴퓨터의 프로그램을 별도 인증 없이 신뢰합니다. 서버는 로컬 Host 이름을 확인하고, 상태를 바꿀 수 있는 요청에 Origin이 있으면 로컬 HTTP 출처만 허용합니다. RPC와 취소 요청은 JSON 본문이어야 합니다. 다른 Host·Origin이나 평문 RPC 요청은 거절하며 내장 화면, Vite 개발 프록시, 로컬 JSON 클라이언트는 그대로 사용할 수 있습니다. 이 검사는 인증을 대신하지 않습니다. 같은 컴퓨터의 다른 프로그램은 Origin을 생략하고 허용된 헤더를 보낼 수 있으므로 막지 못합니다. 심사용(hosted review) 모드는 별도 접근 통제를 사용합니다. diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 2b8ad20..0537338 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -8,6 +8,14 @@ verification state is in [VERIFICATION.md](VERIFICATION.md). 한국어 안내: [KNOWN_LIMITATIONS.ko.md](KNOWN_LIMITATIONS.ko.md) +## Expected results remain predictions + +Expected-result tables and refutation conditions default on only when a project has no saved +choice; saved false values are retained without rewriting the record. Filling missing cells +may require another model call, and larger outputs can consume more tokens. A predicted result +is not an observation. Evidence links do not by themselves validate its causal claim; rows with +no links remain unconfirmed predictions. Existing results are not regenerated by this default. + ## Local access trusts the user of this computer Local mode assumes one user and trusts programs on the same computer without authenticating them. The server checks local Host names and, for requests that can change state, accepts only local HTTP Origins when an Origin is present; RPC and cancellation requests also require JSON. This rejects requests with foreign Host/Origin values and plain-text RPC bodies while preserving the built-in UI, the Vite development proxy and local JSON clients. These checks are not authentication: another program on this computer can omit Origin and send allowed headers. Hosted review has separate access controls. diff --git a/docs/architecture/verification-profiles.md b/docs/architecture/verification-profiles.md index 90ea150..83276fb 100644 --- a/docs/architecture/verification-profiles.md +++ b/docs/architecture/verification-profiles.md @@ -4,6 +4,27 @@ User decision, 2026-10-02: before merge or public release, run one product check source checkout and report its results. The user approves the merge after reviewing those results. During development, retain the focused checks specified in AGENTS.md. +The Product check workflow provides this check on clean Windows runners for pull requests, +main pushes, manual dispatches and the nightly default-branch run (18:00 UTC / 03:00 KST). +The existing `checks` job keeps the static, Web and quick-start checks. `python-fast` collects +everything except `tests/integration`; eight `python-integration` matrix jobs cover every +integration test file exactly once. Each uses `-q -rfE -p no:cacheprovider` plus a JUnit report, +with the same root conftest and inactive-Hook selection policy as a local full run. + +Before merging, require both `checks` and `python-summary` to pass for the reviewed revision. +The summary requires every Python job to succeed and every report to exist; it shows failures, +errors and skips separately. A complete CI result can supply the full product check without +another mandatory local FULL run. Missing, failed or unverified CI coverage does not waive it. +The first rollout must be checked on a real PR before relying on it. Branch-protection settings +are managed separately; editing this workflow does not configure required checks on the server. + +`python scripts/ci_test_shards.py plan` shows the deterministic file assignment and estimated +times. Refresh `tests/ci/integration_durations.json` with +`python scripts/ci_test_shards.py refresh --junit test-results/python.xml` from a complete run. +Only relative file paths and seconds are published. New files join the lightest group using +the median measured file time; stale timing entries do not remove or add tests. The union and +disjointness checks run whenever a shard is selected. Timings are estimates, not timeout changes. + The product check consists of: - Locked Python and Web installation and the README quick start, through the first setup screen. diff --git a/scripts/ci_test_shards.py b/scripts/ci_test_shards.py new file mode 100644 index 0000000..9bd4525 --- /dev/null +++ b/scripts/ci_test_shards.py @@ -0,0 +1,177 @@ +"""Balance integration test files without changing pytest's collection or exclusion policy. + +Refresh timings from a complete JUnit report (including failed tests): + python scripts/ci_test_shards.py refresh --junit test-results/python.xml +Preview all groups or select the JSON file list for one zero-based group: + python scripts/ci_test_shards.py plan --count 8 + python scripts/ci_test_shards.py select --count 8 --index 0 + +Only repository-relative integration paths and elapsed seconds are persisted. New files use +the median positive measured duration (one second if none exist), then join the lightest group. +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import xml.etree.ElementTree as ET +from collections import Counter +from collections.abc import Mapping, Sequence +from pathlib import Path, PurePosixPath +from typing import cast + +ROOT = Path(__file__).resolve().parents[1] +TIMINGS = Path("tests/ci/integration_durations.json") + + +def is_test_path(value: str) -> bool: + path = PurePosixPath(value) + return ( + value == path.as_posix() + and "\\" not in value + and path.parts[:2] == ("tests", "integration") + and ".." not in path.parts + and path.suffix == ".py" + and (path.name.startswith("test_") or path.name.endswith("_test.py")) + ) + + +def discover(root: Path) -> list[str]: + directory = root / "tests/integration" + if not directory.is_dir(): + raise ValueError("integration test directory is missing") + files = sorted( + p.relative_to(root).as_posix() + for p in directory.rglob("*.py") + if p.is_file() and is_test_path(p.relative_to(root).as_posix()) + ) + if not files: + raise ValueError("no integration test files found") + return files + + +def load_durations(path: Path) -> dict[str, float]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("duration table must be a path-to-seconds object") + result: dict[str, float] = {} + for key, value in cast(dict[object, object], raw).items(): + if not isinstance(key, str) or not is_test_path(key): + raise ValueError("duration table contains a non-integration path") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("duration must be a number") + if not math.isfinite(value) or value < 0: + raise ValueError("duration must be finite and nonnegative") + result[key] = float(value) + return result + + +def validate_partition(files: Sequence[str], groups: Sequence[Sequence[str]]) -> None: + counts = Counter(path for group in groups for path in group) + if len(set(files)) != len(files) or set(counts) != set(files): + raise ValueError("shards must cover exactly the discovered integration files") + if any(count != 1 for count in counts.values()): + raise ValueError("an integration file appears in multiple shards") + + +def build_shards( + files: Sequence[str], durations: Mapping[str, float], count: int +) -> tuple[list[list[str]], list[float]]: + if count < 1: + raise ValueError("shard count must be positive") + known = [path for path in files if path in durations] + missing = sorted(set(files) - durations.keys()) + measured = [durations[path] for path in known if durations[path] > 0] + fallback = statistics.median(measured) if measured else 1.0 + groups: list[list[str]] = [[] for _ in range(count)] + loads = [0.0] * count + ordered = [*sorted(known, key=lambda path: (-durations[path], path)), *missing] + for path in ordered: + index = min(range(count), key=lambda index: (loads[index], index)) + groups[index].append(path) + loads[index] += durations.get(path, fallback) + validate_partition(files, groups) + return [sorted(group) for group in groups], loads + + +def durations_from_junit(report: Path, files: Sequence[str]) -> dict[str, float]: + modules = {path[:-3].replace("/", "."): path for path in files} + totals: dict[str, float] = {} + for case in ET.parse(report).getroot().iter("testcase"): + classname = case.get("classname", "") + filename = case.get("file", "").replace("\\", "/") + if filename in files: + path = filename + elif classname.startswith("tests.integration."): + matches = [ + name for name in modules if classname == name or classname.startswith(name + ".") + ] + if not matches: + raise ValueError("integration JUnit case could not be mapped to a test file") + path = modules[max(matches, key=len)] + else: + continue + seconds = float(case.get("time", "0")) + if not math.isfinite(seconds) or seconds < 0: + raise ValueError("JUnit duration must be finite and nonnegative") + totals[path] = totals.get(path, 0.0) + seconds + if not totals: + raise ValueError("JUnit report contains no integration test durations") + return {path: round(seconds, 3) for path, seconds in sorted(totals.items())} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("command", choices=("plan", "select", "refresh")) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--durations", type=Path, default=TIMINGS) + parser.add_argument("--count", type=int, default=8) + parser.add_argument("--index", type=int) + parser.add_argument("--junit", type=Path) + args = parser.parse_args(argv) + try: + files = discover(args.root) + timing_path = args.root / args.durations + if args.command == "refresh": + if args.junit is None: + raise ValueError("refresh requires --junit") + durations = durations_from_junit(args.junit, files) + timing_path.parent.mkdir(parents=True, exist_ok=True) + timing_path.write_text( + json.dumps(durations, indent=2) + "\n", encoding="utf-8", newline="\n" + ) + print(f"Recorded timings for {len(durations)} integration files.") + return 0 + durations = load_durations(timing_path) + groups, loads = build_shards(files, durations, args.count) + if args.command == "select": + if args.index is None or not 0 <= args.index < args.count: + raise ValueError("select requires an index within the shard count") + if not groups[args.index]: + raise ValueError("selected shard is empty; refusing an unscoped pytest run") + print(json.dumps(groups[args.index])) + else: + print( + json.dumps( + { + "file_count": len(files), + "unmeasured_files": sorted(set(files) - durations.keys()), + "shards": [ + {"index": i, "estimated_seconds": round(loads[i], 3), "files": group} + for i, group in enumerate(groups) + ], + }, + indent=2, + ) + ) + except (OSError, ValueError, ET.ParseError) as error: + parser.error(str(error)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_test_summary.py b/scripts/ci_test_summary.py new file mode 100644 index 0000000..8113f43 --- /dev/null +++ b/scripts/ci_test_summary.py @@ -0,0 +1,71 @@ +"""Summarize all Python JUnit reports; incomplete or failed jobs never become a pass.""" + +from __future__ import annotations + +import argparse +import xml.etree.ElementTree as ET +from pathlib import Path + + +def summarize(directory: Path, shards: int, job_results: list[str]) -> tuple[str, bool]: + names = ["python-fast", *(f"python-integration-{i}" for i in range(shards))] + lines = [ + "## Python test results", + "", + "| Job | Tests | Failures | Errors | Skipped |", + "| --- | ---: | ---: | ---: | ---: |", + ] + totals = [0, 0, 0, 0] + problems: list[str] = [] + seen: set[tuple[str, str]] = set() + for name in names: + try: + cases = list(ET.parse(directory / f"{name}.xml").getroot().iter("testcase")) + except (OSError, ET.ParseError): + lines.append(f"| {name} | MISSING/INVALID | — | — | — |") + problems.append(f"Missing or invalid report: {name}") + continue + counts = [ + len(cases), + sum(c.find("failure") is not None for c in cases), + sum(c.find("error") is not None for c in cases), + sum(c.find("skipped") is not None for c in cases), + ] + lines.append(f"| {name} | " + " | ".join(str(n) for n in counts) + " |") + totals = [a + b for a, b in zip(totals, counts, strict=True)] + if not cases: + problems.append(f"Empty report: {name}") + for case in cases: + identity = (case.get("classname", ""), case.get("name", "")) + if identity in seen: + problems.append(f"Duplicate test identity in {name}") + seen.add(identity) + lines.append("| **Total** | " + " | ".join(str(n) for n in totals) + " |") + if len(job_results) != 2 or any(status != "success" for status in job_results): + problems.append("At least one Python job did not succeed.") + if totals[1] or totals[2]: + problems.append("JUnit reports contain failures or errors.") + lines.extend(["", *(f"- {problem}" for problem in dict.fromkeys(problems))]) + lines.extend(["", "Result: " + ("FAIL / incomplete" if problems else "PASS"), ""]) + return "\n".join(lines), not problems + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reports", type=Path, required=True) + parser.add_argument("--shards", type=int, default=8) + parser.add_argument("--job-results", nargs=2, required=True) + parser.add_argument("--markdown-output", type=Path) + args = parser.parse_args() + if args.shards < 1: + parser.error("--shards must be positive") + report, passed = summarize(args.reports, args.shards, args.job_results) + print(report) + if args.markdown_output: + with args.markdown_output.open("a", encoding="utf-8") as stream: + stream.write(report) + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/thoth/application/services/research_attempt_runner.py b/src/thoth/application/services/research_attempt_runner.py index 1b9ee43..e1178c6 100644 --- a/src/thoth/application/services/research_attempt_runner.py +++ b/src/thoth/application/services/research_attempt_runner.py @@ -406,7 +406,7 @@ def _prepare_work( work.auto_retry_interrupted_call = True calls = host.call_settings if calls is not None and calls.hypothesis_contract_v3(request.project_id): - work.hypothesis_contract_v3 = True # the project's own switch, off unless turned on + work.hypothesis_contract_v3 = True # effective project choice, including its default boundary.validate_sources = lambda: host.analysis.require_current_sources( work.evidence, work.context.get("source_context_digest") ) diff --git a/src/thoth/domain/model_call_settings.py b/src/thoth/domain/model_call_settings.py index fff1231..d086cf5 100644 --- a/src/thoth/domain/model_call_settings.py +++ b/src/thoth/domain/model_call_settings.py @@ -10,9 +10,9 @@ # A retry can repeat remote work (no remote stop/idempotency proof). The existing limit remains # at most twice, waiting about 2 s then 4 s; projects can turn it off and resume manually. AUTO_RETRY_INTERRUPTED_DEFAULT = True -# Hypothesis generation contract v3 (what each hypothesis expects from a test, and what would refute -# it): off, so a project keeps the v2 contract exactly until it turns this on. -HYPOTHESIS_CONTRACT_V3_DEFAULT = False +# 사용자 결정 2026-10-23, 가설 v3 실제 비교(12차 3/3, S2 3/3, 10-23 재비교). +# Only an absent choice defaults on; an explicitly saved False keeps the v2 contract. +HYPOTHESIS_CONTRACT_V3_DEFAULT = True class ProjectModelCallSettings(DomainModel): diff --git a/tests/ci/integration_durations.json b/tests/ci/integration_durations.json new file mode 100644 index 0000000..2d27935 --- /dev/null +++ b/tests/ci/integration_durations.json @@ -0,0 +1,363 @@ +{ + "tests/integration/test_6g_full_backend_cycle.py": 12.681, + "tests/integration/test_a01_storage_atomicity.py": 64.941, + "tests/integration/test_a02_autonomous_acquisition.py": 22.256, + "tests/integration/test_a02_real_connector_routes.py": 12.183, + "tests/integration/test_a03_critical_counter_search.py": 83.319, + "tests/integration/test_a03_multi_wave_counter_search.py": 61.121, + "tests/integration/test_a03_portfolio_atomicity.py": 97.537, + "tests/integration/test_a03_storage_atomicity.py": 143.743, + "tests/integration/test_a04_batch_atomicity.py": 10.623, + "tests/integration/test_a04_managed_normal_thread.py": 39.956, + "tests/integration/test_a04_r2_closed_loop.py": 22.022, + "tests/integration/test_a04_runtime_router.py": 0.003, + "tests/integration/test_a04_stale_evidence.py": 76.057, + "tests/integration/test_a04_storage_authority.py": 82.28, + "tests/integration/test_a05_bounded_recovery.py": 33.176, + "tests/integration/test_a06_codex_memory_review.py": 32.032, + "tests/integration/test_a06_full_project_memory.py": 44.032, + "tests/integration/test_a06_memory_branch_handoff.py": 23.154, + "tests/integration/test_a06_memory_preparation.py": 30.367, + "tests/integration/test_a06_memory_storage_boundary.py": 93.192, + "tests/integration/test_a06_semantic_memory_review.py": 0.001, + "tests/integration/test_a07_semantic_three_way_merge.py": 83.437, + "tests/integration/test_a08_artifact_rollback.py": 5.216, + "tests/integration/test_a08_prompt_bundle_candidate.py": 8.646, + "tests/integration/test_a08_recursive_improvement.py": 101.767, + "tests/integration/test_a08_retrieval_policy_candidate.py": 3.371, + "tests/integration/test_a08_storage_authority.py": 172.556, + "tests/integration/test_a08_workflow_candidate_canary.py": 12.37, + "tests/integration/test_a09_all_profile_packs.py": 55.332, + "tests/integration/test_a09_ambiguous_profile_hold.py": 2.8, + "tests/integration/test_a09_dynamic_criterion_profile.py": 19.18, + "tests/integration/test_a09_profile_version_upgrade.py": 7.24, + "tests/integration/test_a09_storage_atomicity.py": 83.209, + "tests/integration/test_a10_storage_authority.py": 111.567, + "tests/integration/test_a10_unified_receipt_dag.py": 26.95, + "tests/integration/test_a11_policy_integrity.py": 53.676, + "tests/integration/test_a12_blind_score_and_adjudication.py": 1.264, + "tests/integration/test_a12_counterbalanced_assignment.py": 2.013, + "tests/integration/test_a12_field_measurement_tooling.py": 13.392, + "tests/integration/test_a12_identity_leakage.py": 0.924, + "tests/integration/test_a12_session_roundtrip.py": 7.025, + "tests/integration/test_a13_authenticated_multiplayer.py": 18.841, + "tests/integration/test_a13_operation_scope.py": 161.08, + "tests/integration/test_a13_project_atomicity.py": 79.583, + "tests/integration/test_a13_security_boundaries.py": 14.143, + "tests/integration/test_acquisition_io_transaction_boundary.py": 6.962, + "tests/integration/test_action_effect_value_type.py": 21.069, + "tests/integration/test_action_effort_estimates.py": 58.377, + "tests/integration/test_action_full_rpc.py": 9.206, + "tests/integration/test_action_query_readonly.py": 5.324, + "tests/integration/test_action_response_golden.py": 96.88, + "tests/integration/test_action_semantic_uow.py": 8.985, + "tests/integration/test_action_step_add_step_id.py": 127.668, + "tests/integration/test_active_behavior_consumed.py": 49.902, + "tests/integration/test_approval_clock_rollback.py": 14.919, + "tests/integration/test_architecture_cleanup_projectpack_migration.py": 33.677, + "tests/integration/test_artifact_row_codec_contract.py": 11.096, + "tests/integration/test_authorization_consumption_guards.py": 16.15, + "tests/integration/test_automatic_behavior_candidate.py": 20.135, + "tests/integration/test_behavior_exposure_authority.py": 19.337, + "tests/integration/test_behavior_exposure_boundaries.py": 47.902, + "tests/integration/test_behavior_exposure_recovery.py": 21.937, + "tests/integration/test_behavior_exposure_runtime.py": 77.5, + "tests/integration/test_behavior_policy_comparison.py": 12.233, + "tests/integration/test_claude_code_login_entry.py": 10.595, + "tests/integration/test_claude_code_normal_entry.py": 7.352, + "tests/integration/test_claude_oauth_normal_entry.py": 9.4, + "tests/integration/test_claude_runtime_owner.py": 3.195, + "tests/integration/test_closure_open_item_contract.py": 36.791, + "tests/integration/test_codex_auth_nonblocking.py": 10.07, + "tests/integration/test_confirmed_findings_result.py": 21.068, + "tests/integration/test_connector_sandbox_cycle.py": 3.981, + "tests/integration/test_conversation_ui_read.py": 17.95, + "tests/integration/test_counter_cleanup_cancel_preserves_prior.py": 10.307, + "tests/integration/test_counter_clock_sequence.py": 20.507, + "tests/integration/test_criteria_full_rpc.py": 6.227, + "tests/integration/test_current_hero_normal_entry.py": 26.283, + "tests/integration/test_current_secondary_same_trace.py": 55.955, + "tests/integration/test_cutoff_precision_currentness.py": 41.044, + "tests/integration/test_cycle_criterion_staging_contract.py": 72.323, + "tests/integration/test_cycle_model_phase_contract.py": 65.781, + "tests/integration/test_d1_object_followup_atomicity.py": 95.916, + "tests/integration/test_d2_counterevidence_stage.py": 12.08, + "tests/integration/test_decision_object_restore_fail_closed.py": 26.887, + "tests/integration/test_discovery_cleanup_cancel.py": 7.288, + "tests/integration/test_evaluation_execution_limits.py": 19.152, + "tests/integration/test_evaluation_pair_lifecycle.py": 82.733, + "tests/integration/test_evaluation_producer_binding.py": 58.62, + "tests/integration/test_evaluation_thread_scope.py": 12.764, + "tests/integration/test_evidence_coverage.py": 53.951, + "tests/integration/test_evidence_full_rpc.py": 6.811, + "tests/integration/test_execution_call_order.py": 125.398, + "tests/integration/test_execution_full_rpc.py": 11.5, + "tests/integration/test_execution_not_applied_dispatch.py": 46.985, + "tests/integration/test_four_projectpack_portability.py": 19.621, + "tests/integration/test_full_backend_adversarial_isolation.py": 3.481, + "tests/integration/test_full_backend_soak.py": 22.613, + "tests/integration/test_full_thread_cycle.py": 2.565, + "tests/integration/test_hero_overlay_and_baseline_preview.py": 28.349, + "tests/integration/test_hidden_holdout_oracle_evaluator.py": 21.321, + "tests/integration/test_historical_access_verification.py": 42.022, + "tests/integration/test_history_change_summary_read.py": 7.152, + "tests/integration/test_hosted_review_dispatch_gate.py": 28.397, + "tests/integration/test_hosted_review_internet_vs_model.py": 11.773, + "tests/integration/test_hosted_review_session_isolation.py": 32.977, + "tests/integration/test_hosted_review_stale_attempt.py": 25.308, + "tests/integration/test_hypothesis_action_guard.py": 84.915, + "tests/integration/test_hypothesis_binding_draft_contract.py": 16.872, + "tests/integration/test_hypothesis_contract_v3_run.py": 81.685, + "tests/integration/test_hypothesis_full_rpc.py": 9.375, + "tests/integration/test_hypothesis_link.py": 95.346, + "tests/integration/test_hypothesis_prediction_outcome_cycle.py": 158.847, + "tests/integration/test_hypothesis_same.py": 95.454, + "tests/integration/test_hypothesis_v4_run.py": 58.347, + "tests/integration/test_improvement_exposure_golden.py": 88.823, + "tests/integration/test_improvement_full_rpc.py": 4.446, + "tests/integration/test_improvement_handlers_golden.py": 86.12, + "tests/integration/test_improvement_holdout_exposure.py": 11.093, + "tests/integration/test_ingestion_service.py": 3.412, + "tests/integration/test_investigation_full_rpc.py": 4.473, + "tests/integration/test_journal_completion_concurrency.py": 9.289, + "tests/integration/test_journal_completion_faults.py": 32.461, + "tests/integration/test_journal_completion_process.py": 19.254, + "tests/integration/test_journal_completion_replay.py": 39.8, + "tests/integration/test_judgment_records.py": 66.167, + "tests/integration/test_judgment_review.py": 52.86, + "tests/integration/test_language_revision_roundtrip.py": 17.773, + "tests/integration/test_lesson_recall.py": 43.016, + "tests/integration/test_lifecycle_rpc.py": 7.345, + "tests/integration/test_local_first_run_model_boundary.py": 28.565, + "tests/integration/test_local_http_origin_baseline.py": 54.324, + "tests/integration/test_local_workspace_restart.py": 13.666, + "tests/integration/test_m2_counter_cancel_boundary.py": 8.363, + "tests/integration/test_m2_counter_publication_boundaries.py": 34.909, + "tests/integration/test_m2_cycle_publication_boundaries.py": 58.622, + "tests/integration/test_m2_evidence_publication_boundaries.py": 26.692, + "tests/integration/test_m2_result_publication_boundaries.py": 43.724, + "tests/integration/test_m3_action_authorization_atomicity.py": 43.961, + "tests/integration/test_m3_action_mutation_atomicity.py": 156.745, + "tests/integration/test_m3_direct_test_binding_contract.py": 67.358, + "tests/integration/test_m3_execution_effect_atomicity.py": 145.257, + "tests/integration/test_m3_execution_revision_atomicity.py": 129.762, + "tests/integration/test_m3_hypothesis_auxiliary_atomicity.py": 38.314, + "tests/integration/test_m3_hypothesis_first_write_reachability.py": 31.931, + "tests/integration/test_m3_hypothesis_mutation_atomicity.py": 60.26, + "tests/integration/test_m3_hypothesis_proposal_audit_atomicity.py": 7.435, + "tests/integration/test_m3_hypothesis_publication_atomicity.py": 63.387, + "tests/integration/test_m3_object_creation_atomicity.py": 13.364, + "tests/integration/test_m3_object_mutation_atomicity.py": 96.814, + "tests/integration/test_m3_outcome_child_atomicity.py": 68.771, + "tests/integration/test_m3_outcome_revision_atomicity.py": 105.138, + "tests/integration/test_m3_prediction_preparation_atomicity.py": 145.062, + "tests/integration/test_m3_substantive_appraisal_atomicity.py": 130.979, + "tests/integration/test_m4_memory_control_multiwrite_observation.py": 40.904, + "tests/integration/test_m5_exposure_service_atomicity.py": 49.933, + "tests/integration/test_m5_improvement_handlers_atomicity.py": 60.302, + "tests/integration/test_m5_r2_closed_loop_atomicity.py": 45.376, + "tests/integration/test_m5_recursive_improvement_atomicity.py": 57.168, + "tests/integration/test_m5_reference_answer_atomicity.py": 324.022, + "tests/integration/test_m5_revision_control_atomicity.py": 15.711, + "tests/integration/test_memory_edit.py": 185.733, + "tests/integration/test_memory_full_rpc.py": 4.681, + "tests/integration/test_memory_handler_golden.py": 115.009, + "tests/integration/test_memory_query_expansion.py": 10.649, + "tests/integration/test_memory_selection.py": 46.017, + "tests/integration/test_memory_service_golden.py": 89.236, + "tests/integration/test_memory_store.py": 2.494, + "tests/integration/test_migration_logging.py": 7.232, + "tests/integration/test_migrations.py": 13.846, + "tests/integration/test_model_call_hold_failure.py": 20.783, + "tests/integration/test_model_call_settings.py": 38.374, + "tests/integration/test_model_catalog_refresh_rpc.py": 8.217, + "tests/integration/test_model_catalog_startup.py": 12.631, + "tests/integration/test_model_execution_marks.py": 44.814, + "tests/integration/test_model_input_admission.py": 4.362, + "tests/integration/test_model_settings.py": 28.404, + "tests/integration/test_model_settings_cache_only.py": 14.893, + "tests/integration/test_model_tooling_rpc.py": 10.821, + "tests/integration/test_model_wait_lease.py": 9.364, + "tests/integration/test_multi_baseline_normal_thread.py": 29.348, + "tests/integration/test_normal_public_research_identity.py": 134.292, + "tests/integration/test_object_full_rpc.py": 11.219, + "tests/integration/test_object_preclaim_contract.py": 5.273, + "tests/integration/test_operation_cancel.py": 3.484, + "tests/integration/test_operation_claim_busy.py": 13.868, + "tests/integration/test_operation_claim_concurrency.py": 20.617, + "tests/integration/test_operation_journal.py": 6.431, + "tests/integration/test_operation_journal_failure_boundaries.py": 11.406, + "tests/integration/test_operation_owner_atomicity.py": 14.952, + "tests/integration/test_operation_owner_completion_race.py": 27.729, + "tests/integration/test_operation_owner_migration.py": 9.484, + "tests/integration/test_operation_owner_recovery.py": 32.379, + "tests/integration/test_operation_owner_storage.py": 27.669, + "tests/integration/test_operation_started_atomic.py": 45.788, + "tests/integration/test_outcome_full_rpc.py": 9.263, + "tests/integration/test_outcome_response_golden.py": 80.355, + "tests/integration/test_p_memory_atomicity.py": 112.922, + "tests/integration/test_p_r2_failure_atomicity.py": 357.247, + "tests/integration/test_p_reference_failure_notice.py": 134.184, + "tests/integration/test_paired_evaluation_execution.py": 42.229, + "tests/integration/test_post_audit_runtime.py": 20.296, + "tests/integration/test_post_execution_memory_recovery.py": 48.665, + "tests/integration/test_post_learning_running_recovery.py": 213.626, + "tests/integration/test_prepare_synthetic_radar_demo.py": 32.194, + "tests/integration/test_profile_cross_domain_overfit.py": 37.72, + "tests/integration/test_project_governance_rpc.py": 4.102, + "tests/integration/test_project_list_reads.py": 45.409, + "tests/integration/test_project_restore_rpc.py": 50.281, + "tests/integration/test_projectpack_rpc.py": 7.273, + "tests/integration/test_projectpack_run.py": 7.714, + "tests/integration/test_projectpack_source_binding.py": 17.617, + "tests/integration/test_protected_authorization.py": 49.159, + "tests/integration/test_psyche_connected_sources_live.py": 0.0, + "tests/integration/test_psyche_workspace_snapshot.py": 10.397, + "tests/integration/test_public_browser.py": 18.233, + "tests/integration/test_public_browser_cleanup.py": 3.04, + "tests/integration/test_public_browser_navigation.py": 22.142, + "tests/integration/test_public_oauth_memory_excerpt_regression.py": 30.417, + "tests/integration/test_public_web_evidence_gates.py": 41.845, + "tests/integration/test_public_web_policy_update_rpc.py": 33.933, + "tests/integration/test_public_web_runtime_catalog.py": 7.116, + "tests/integration/test_r2_closed_loop_golden.py": 317.931, + "tests/integration/test_r2_replay_guard.py": 132.116, + "tests/integration/test_real_exposure_lifecycle.py": 3.95, + "tests/integration/test_receipt_closure_export_full_rpc.py": 5.012, + "tests/integration/test_recursive_improvement_golden.py": 104.57, + "tests/integration/test_reference_approved_formula.py": 12.791, + "tests/integration/test_reference_calculation_roundtrip.py": 24.969, + "tests/integration/test_reference_input_boundaries.py": 99.812, + "tests/integration/test_reference_mapping_boundary.py": 74.588, + "tests/integration/test_reference_model_mapping.py": 25.534, + "tests/integration/test_reference_range_interview.py": 7.466, + "tests/integration/test_reference_scenarios.py": 14.647, + "tests/integration/test_reference_service_golden.py": 217.61, + "tests/integration/test_reference_source_changes.py": 50.658, + "tests/integration/test_reference_source_rights.py": 9.087, + "tests/integration/test_reference_storage_boundary.py": 69.226, + "tests/integration/test_registered_site_discovery.py": 11.313, + "tests/integration/test_request_response_language.py": 70.214, + "tests/integration/test_research_activity_read_contract.py": 61.05, + "tests/integration/test_research_assembly_cache_basis.py": 5.661, + "tests/integration/test_research_boundaries_v2.py": 120.088, + "tests/integration/test_research_catalog_usage.py": 46.62, + "tests/integration/test_research_cycle_restore.py": 155.292, + "tests/integration/test_research_discovery_contract.py": 99.319, + "tests/integration/test_research_export_grounding.py": 27.915, + "tests/integration/test_research_failure_terminal.py": 29.368, + "tests/integration/test_research_final_repairs.py": 9.32, + "tests/integration/test_research_focus_basis.py": 68.611, + "tests/integration/test_research_followup_read.py": 33.333, + "tests/integration/test_research_history_read.py": 35.428, + "tests/integration/test_research_identity_migration.py": 3.881, + "tests/integration/test_research_improvement_observation.py": 26.801, + "tests/integration/test_research_input_queue.py": 160.596, + "tests/integration/test_research_lineage_publication.py": 42.524, + "tests/integration/test_research_memory_basis.py": 13.331, + "tests/integration/test_research_memory_correction_currentness.py": 71.558, + "tests/integration/test_research_process_and_queries.py": 8.325, + "tests/integration/test_research_publication_guards.py": 62.229, + "tests/integration/test_research_query_egress.py": 26.641, + "tests/integration/test_research_readset.py": 94.95, + "tests/integration/test_research_readset_memory.py": 32.582, + "tests/integration/test_research_recovery.py": 21.172, + "tests/integration/test_research_registered_web_route.py": 22.223, + "tests/integration/test_research_replay_ownership.py": 56.957, + "tests/integration/test_research_request_v2.py": 37.963, + "tests/integration/test_research_review_order.py": 10.614, + "tests/integration/test_research_scoped_conflicts.py": 16.613, + "tests/integration/test_research_source_queries_contract.py": 54.679, + "tests/integration/test_research_usage_read_order.py": 59.114, + "tests/integration/test_resource_scope_atomicity.py": 22.736, + "tests/integration/test_resource_scope_derived.py": 40.375, + "tests/integration/test_resource_scope_execution.py": 4.348, + "tests/integration/test_resource_scope_export.py": 37.804, + "tests/integration/test_resource_scope_integrity.py": 11.76, + "tests/integration/test_resource_scope_investigation.py": 11.398, + "tests/integration/test_resource_scope_legacy_transfer.py": 5.802, + "tests/integration/test_resource_scope_memory_b.py": 33.404, + "tests/integration/test_resource_scope_memory_controls.py": 25.245, + "tests/integration/test_resource_scope_memory_preparation.py": 10.971, + "tests/integration/test_resource_scope_model_b.py": 16.58, + "tests/integration/test_resource_scope_objects.py": 14.124, + "tests/integration/test_resource_scope_receipt_correction.py": 7.176, + "tests/integration/test_resource_scope_records_golden.py": 49.702, + "tests/integration/test_resource_scope_research_b.py": 16.692, + "tests/integration/test_resource_scope_revocation.py": 28.68, + "tests/integration/test_resource_scope_roundtrip.py": 23.816, + "tests/integration/test_resource_scope_sandbox_b.py": 13.199, + "tests/integration/test_resource_scope_transition_boundaries.py": 32.625, + "tests/integration/test_restore_access.py": 8.276, + "tests/integration/test_restore_apply_atomicity.py": 172.178, + "tests/integration/test_restore_concurrency.py": 19.515, + "tests/integration/test_restore_currentness_consumers.py": 21.307, + "tests/integration/test_restore_drift.py": 23.297, + "tests/integration/test_restore_legacy_compatibility.py": 21.277, + "tests/integration/test_restore_preview_contract.py": 11.502, + "tests/integration/test_restore_service.py": 4.633, + "tests/integration/test_restore_source_basis.py": 10.807, + "tests/integration/test_restore_typed_references.py": 32.313, + "tests/integration/test_result_compare_criteria.py": 15.791, + "tests/integration/test_result_usage_read.py": 14.671, + "tests/integration/test_resume_interrupted_research.py": 75.491, + "tests/integration/test_retrieval_gap_reassessment.py": 17.414, + "tests/integration/test_revision_command_golden.py": 39.402, + "tests/integration/test_revision_full_rpc.py": 6.577, + "tests/integration/test_revision_merge_late_write_observation.py": 14.838, + "tests/integration/test_revision_publication_golden.py": 51.99, + "tests/integration/test_revision_publication_success_golden.py": 13.785, + "tests/integration/test_revision_read_bindings.py": 5.545, + "tests/integration/test_revision_remaining_publication_observations.py": 40.484, + "tests/integration/test_revision_rpc.py": 5.799, + "tests/integration/test_revision_service.py": 33.263, + "tests/integration/test_runtime_acquisition_composition.py": 78.647, + "tests/integration/test_runtime_decision_composition.py": 88.159, + "tests/integration/test_runtime_entry_assembly.py": 61.897, + "tests/integration/test_runtime_entry_failure_cleanup.py": 71.026, + "tests/integration/test_runtime_registration.py": 33.507, + "tests/integration/test_runtime_registration_table.py": 24.006, + "tests/integration/test_runtime_research_composition.py": 79.567, + "tests/integration/test_runtime_sandbox_composition.py": 83.565, + "tests/integration/test_schema_family_factories.py": 18.488, + "tests/integration/test_scope_read_memo.py": 39.281, + "tests/integration/test_scope_read_request_memo.py": 9.376, + "tests/integration/test_selected_evidence_query.py": 5.394, + "tests/integration/test_source_cleanup_cancel_boundary.py": 42.209, + "tests/integration/test_source_list_cost.py": 2.916, + "tests/integration/test_source_rpc.py": 42.664, + "tests/integration/test_source_time_connection.py": 20.438, + "tests/integration/test_stage_checkpoint_contract.py": 15.86, + "tests/integration/test_stale_running_current_behavior.py": 39.47, + "tests/integration/test_state_transition_noop.py": 20.403, + "tests/integration/test_state_transition_repeat_current_behavior.py": 98.992, + "tests/integration/test_storage_clock_currentness.py": 11.384, + "tests/integration/test_storage_migration_ddl_contract.py": 20.14, + "tests/integration/test_storage_schema_registration_contract.py": 22.682, + "tests/integration/test_store_factory_runtime.py": 2.715, + "tests/integration/test_structured_context_pipeline.py": 18.134, + "tests/integration/test_thread_analysis_failure_diagnostics.py": 10.374, + "tests/integration/test_thread_cycle_preparation_contract.py": 20.669, + "tests/integration/test_thread_full_catalog_rpc.py": 6.649, + "tests/integration/test_thread_read_currentness_cost.py": 26.201, + "tests/integration/test_thread_read_summary_view.py": 31.112, + "tests/integration/test_thread_read_view_order.py": 48.415, + "tests/integration/test_thread_rpc.py": 3.142, + "tests/integration/test_trace_csv.py": 113.036, + "tests/integration/test_trace_csv_closure_column.py": 20.821, + "tests/integration/test_trace_origin.py": 100.337, + "tests/integration/test_trace_origin_siblings.py": 57.708, + "tests/integration/test_transport_diagnostic_persistence.py": 13.953, + "tests/integration/test_tui_auto_context.py": 13.238, + "tests/integration/test_tui_full_local_cycle.py": 9.613, + "tests/integration/test_tui_prompt_injection.py": 0.0, + "tests/integration/test_tui_resume_after_restart.py": 2.96, + "tests/integration/test_u06_research_roundtrip.py": 108.55, + "tests/integration/test_user_activity_thread_read.py": 9.731, + "tests/integration/test_v2_cycle_lineage_values.py": 6.028, + "tests/integration/test_workspace_owner_pin.py": 0.025, + "tests/integration/test_workspace_restart_storage.py": 63.336, + "tests/integration/test_workspace_runtime_exclusivity_baseline.py": 32.375, + "tests/integration/test_workspace_runtime_lock.py": 50.773, + "tests/integration/test_xai_workspace_route.py": 23.182 +} diff --git a/tests/ci/test_ci_test_shards.py b/tests/ci/test_ci_test_shards.py new file mode 100644 index 0000000..996b476 --- /dev/null +++ b/tests/ci/test_ci_test_shards.py @@ -0,0 +1,137 @@ +"""The CI partition includes each real integration file exactly once.""" + +import json +from pathlib import Path + +import pytest +from scripts.ci_test_shards import ( + build_shards, + discover, + durations_from_junit, + load_durations, + main, + validate_partition, +) + + +def test_largest_files_are_balanced_deterministically() -> None: + times = {f"tests/integration/test_{i}.py": n for i, n in enumerate([90, 80, 70, 60, 50, 40])} + groups, loads = build_shards(list(times), times, 3) + assert loads == [130, 130, 130] + assert build_shards(list(reversed(times)), dict(reversed(list(times.items()))), 3) == ( + groups, + loads, + ) + validate_partition(list(times), groups) + + +def test_new_file_joins_lightest_group_and_removed_timing_is_ignored() -> None: + times = { + "tests/integration/test_long.py": 20.0, + "tests/integration/test_short.py": 5.0, + "tests/integration/test_deleted.py": 999.0, + } + files = [ + "tests/integration/test_long.py", + "tests/integration/test_short.py", + "tests/integration/test_new.py", + ] + groups, loads = build_shards(files, times, 2) + assert groups[1] == ["tests/integration/test_new.py", "tests/integration/test_short.py"] + assert loads == [20.0, 17.5] + validate_partition(files, groups) + + +def test_all_new_files_have_a_positive_balancing_estimate() -> None: + files = [f"tests/integration/test_{i}.py" for i in range(8)] + groups, loads = build_shards(files, {}, 8) + assert loads == [1.0] * 8 and all(len(group) == 1 for group in groups) + + +@pytest.mark.parametrize("groups", [[["a"], []], [["a"], ["a", "b"]], [["a", "b", "c"]]]) +def test_missing_duplicate_or_extra_file_is_rejected(groups: list[list[str]]) -> None: + with pytest.raises(ValueError): + validate_partition(["a", "b"], groups) + + +def test_discovery_uses_both_pytest_filename_patterns_and_nested_directories( + tmp_path: Path, +) -> None: + directory = tmp_path / "tests/integration/nested" + directory.mkdir(parents=True) + for name in ("test_alpha.py", "beta_test.py", "helper.py", "conftest.py"): + (directory / name).write_text("", encoding="utf-8") + assert discover(tmp_path) == [ + "tests/integration/nested/beta_test.py", + "tests/integration/nested/test_alpha.py", + ] + + +@pytest.mark.parametrize( + "table", + [ + [], + {"/private/test_bad.py": 1}, + {"tests/integration/../test_bad.py": 1}, + {"tests/integration/test_a.py": -1}, + {"tests/integration/test_a.py": float("nan")}, + {"tests/integration/test_a.py": True}, + ], +) +def test_bad_duration_metadata_is_not_silently_used(tmp_path: Path, table: object) -> None: + path = tmp_path / "durations.json" + path.write_text(json.dumps(table), encoding="utf-8") + with pytest.raises(ValueError): + load_durations(path) + + +def test_junit_refresh_counts_failed_cases_and_class_methods_without_private_paths( + tmp_path: Path, +) -> None: + report = tmp_path / "report.xml" + report.write_text( + """ + + + + + + """, + encoding="utf-8", + ) + assert durations_from_junit(report, ["tests/integration/test_a.py"]) == { + "tests/integration/test_a.py": 5.5 + } + with pytest.raises(ValueError, match="mapped"): + durations_from_junit(report, ["tests/integration/test_other.py"]) + directory = tmp_path / "tests/integration" + directory.mkdir(parents=True) + (directory / "test_a.py").write_text("", encoding="utf-8") + assert main(["refresh", "--root", str(tmp_path), "--junit", str(report)]) == 0 + persisted = (tmp_path / "tests/ci/integration_durations.json").read_bytes() + assert b"\r\n" not in persisted + assert json.loads(persisted) == {"tests/integration/test_a.py": 5.5} + + +def test_select_revalidates_actual_files_and_refuses_empty_shard( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + directory = tmp_path / "tests/integration" + directory.mkdir(parents=True) + (directory / "test_new.py").write_text("", encoding="utf-8") + timings = tmp_path / "durations.json" + timings.write_text("{}", encoding="utf-8") + args = ["select", "--root", str(tmp_path), "--durations", str(timings), "--count", "2"] + assert main([*args, "--index", "0"]) == 0 + assert json.loads(capsys.readouterr().out) == ["tests/integration/test_new.py"] + with pytest.raises(SystemExit) as error: + main([*args, "--index", "1"]) + assert error.value.code == 2 + + +def test_real_repository_partition_is_complete_and_disjoint() -> None: + root = Path(__file__).resolve().parents[2] + files = discover(root) + groups, _ = build_shards(files, load_durations(root / "tests/ci/integration_durations.json"), 8) + validate_partition(files, groups) + assert sum(map(len, groups)) == len(files) diff --git a/tests/ci/test_ci_test_summary.py b/tests/ci/test_ci_test_summary.py new file mode 100644 index 0000000..2ba333b --- /dev/null +++ b/tests/ci/test_ci_test_summary.py @@ -0,0 +1,56 @@ +"""Missing artifacts, failed jobs and duplicate tests cannot produce a passing summary.""" + +from pathlib import Path + +import pytest +from scripts.ci_test_summary import summarize + + +def write_report(path: Path, name: str, detail: str = "") -> None: + path.write_text( + f'' + f"{detail}", + encoding="utf-8", + ) + + +def test_summary_counts_every_report_and_keeps_skips_visible(tmp_path: Path) -> None: + write_report(tmp_path / "python-fast.xml", "fast", "") + for i in range(8): + write_report(tmp_path / f"python-integration-{i}.xml", f"integration-{i}") + report, passed = summarize(tmp_path, 8, ["success", "success"]) + assert passed and "| **Total** | 9 | 0 | 0 | 1 |" in report + assert all(f"python-integration-{i}" in report for i in range(8)) + + +@pytest.mark.parametrize("status", ["failure", "cancelled", "skipped"]) +def test_job_failure_is_not_overridden_by_passing_xml(tmp_path: Path, status: str) -> None: + write_report(tmp_path / "python-fast.xml", "fast") + write_report(tmp_path / "python-integration-0.xml", "integration") + _, passed = summarize(tmp_path, 1, ["success", status]) + assert not passed + + +@pytest.mark.parametrize("detail", ["", ""]) +def test_junit_failure_is_not_overridden_by_job_status(tmp_path: Path, detail: str) -> None: + write_report(tmp_path / "python-fast.xml", "fast", detail) + write_report(tmp_path / "python-integration-0.xml", "integration") + report, passed = summarize(tmp_path, 1, ["success", "success"]) + assert not passed and "failures or errors" in report + + +def test_missing_invalid_and_empty_reports_fail(tmp_path: Path) -> None: + (tmp_path / "python-fast.xml").write_text("bad XML", encoding="utf-8") + report, passed = summarize(tmp_path, 1, ["success", "success"]) + assert not passed and "MISSING/INVALID" in report + (tmp_path / "python-fast.xml").write_text("", encoding="utf-8") + write_report(tmp_path / "python-integration-0.xml", "integration") + report, passed = summarize(tmp_path, 1, ["success", "success"]) + assert not passed and "Empty report" in report + + +def test_cross_shard_duplicate_is_a_failure(tmp_path: Path) -> None: + write_report(tmp_path / "python-fast.xml", "same") + write_report(tmp_path / "python-integration-0.xml", "same") + report, passed = summarize(tmp_path, 1, ["success", "success"]) + assert not passed and "Duplicate test identity" in report diff --git a/tests/integration/test_hypothesis_contract_v3_run.py b/tests/integration/test_hypothesis_contract_v3_run.py index 85453ae..0c84d64 100644 --- a/tests/integration/test_hypothesis_contract_v3_run.py +++ b/tests/integration/test_hypothesis_contract_v3_run.py @@ -1,4 +1,4 @@ -"""Hypothesis generation contract v3: off by default, the same calls when on, and a settled record. +"""Hypothesis contract v3: default on, saved v2 choice, and settled records. A project turns it on in its model-call settings. When on, the one generator call is shown the v3 form and contract text, what it says about expected results and refutation conditions is stored @@ -14,11 +14,13 @@ from typing import Any import pytest +from tests.integration.scoped_runtime import fixture_scope_policy from tests.integration.test_hypothesis_link import investigate from tests.integration.test_research_request_v2 import ControlledResearchModel from tests.integration.test_trace_origin import RAIN, Rpc, opened from thoth.application.services.hypothesis_projection import hypothesis_view +from thoth.apps.runtime import create_runtime from thoth.domain.canonical import model_digest from thoth.domain.enums import ModelRole from thoth.domain.hypothesis import ( @@ -145,6 +147,14 @@ async def run( runtime, rpc = await opened(tmp_path, model) if on: await turn_on(rpc) + else: + current = await settings(rpc) + await rpc( + "model/callSettings/update", + project_id="p", + hypothesis_contract_v3=False, + expected_digest=current["settings_digest"], + ) thread_id = await investigate(runtime, rpc, RAIN) return runtime, model, rpc, thread_id, model @@ -155,13 +165,13 @@ async def portfolio_of(rpc: Rpc, thread_id: str) -> dict[str, Any]: @pytest.mark.asyncio -async def test_the_switch_is_off_by_default_and_each_switch_keeps_the_other(tmp_path: Path) -> None: +async def test_the_switch_is_on_by_default_and_each_switch_keeps_the_other(tmp_path: Path) -> None: runtime, rpc = await opened(tmp_path, GeneratorModel()) try: first = await settings(rpc) - # 사용자 결정 20261010-01: only retry defaults on; v3 still defaults off. + # 사용자 결정 2026-10-23: both absent choices default on. assert (first["hypothesis_contract_v3"], first["auto_retry_interrupted_model_call"]) == ( - False, + True, True, ) await turn_on(rpc) @@ -185,6 +195,41 @@ async def test_the_switch_is_off_by_default_and_each_switch_keeps_the_other(tmp_ runtime.close() +@pytest.mark.parametrize("saved", [None, False, True]) +async def test_choice_survives_reopen_without_rewrite_and_drives_generator( + tmp_path: Path, saved: bool | None +) -> None: + runtime, rpc = await opened(tmp_path, GeneratorModel()) + try: + if saved is not None: + await rpc( + "model/callSettings/update", + project_id="p", + hypothesis_contract_v3=saved, + expected_digest=None, + ) + before = await settings(rpc) + assert before["hypothesis_contract_v3"] is (True if saved is None else saved) + assert (before["settings_digest"] is None) is (saved is None) + count = rpc.count + finally: + runtime.close() + model = GeneratorModel() + runtime = create_runtime( + tmp_path, model_resolver=model, resource_scope_policy=fixture_scope_policy() + ) + rpc = Rpc(runtime) + rpc.count = count + try: + assert await settings(rpc) == before + await investigate(runtime, rpc, RAIN) + (call,) = generator_calls(model) + assert call.prompt_version == (CONTRACT_V2 if saved is False else CONTRACT_V3) + assert await settings(rpc) == before # no default materialization or saved-choice rewrite + finally: + runtime.close() + + @pytest.mark.asyncio async def test_off_the_generator_sees_v2_and_nothing_of_v3_is_stored_or_shown( tmp_path: Path, diff --git a/tests/integration/test_model_call_settings.py b/tests/integration/test_model_call_settings.py index b7cb1bd..a1b27c4 100644 --- a/tests/integration/test_model_call_settings.py +++ b/tests/integration/test_model_call_settings.py @@ -57,7 +57,7 @@ async def call(method: str, key: str, **more: JsonValue) -> Json: # 사용자 결정 20261010-01: an unsaved project starts on. assert first == { "auto_retry_interrupted_model_call": True, - "hypothesis_contract_v3": False, + "hypothesis_contract_v3": True, "settings_digest": None, } on = await call( From 258c6974c6f93ce534084f8d17efc3ece10dd7cb Mon Sep 17 00:00:00 2001 From: THOTH Date: Fri, 23 Oct 2026 18:37:39 +0900 Subject: [PATCH 2/3] Keep the CLI help contract test independent of CI terminal flags On GitHub runners GITHUB_ACTIONS makes Typer force terminal mode and ignore COLUMNS=120. The contract test and capture script now remove GITHUB_ACTIONS, FORCE_COLOR, PY_COLORS and TERMINAL_WIDTH before rendering help. The recorded contract is unchanged. --- SOURCE_MANIFEST.json | 10 +++++----- tests/cli_contract_capture.py | 18 ++++++++++++++---- tests/test_cli_registration_contract.py | 5 ++++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/SOURCE_MANIFEST.json b/SOURCE_MANIFEST.json index 6de503b..d72c4f3 100644 --- a/SOURCE_MANIFEST.json +++ b/SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schema_version": "thoth-source-preview-manifest-v1", - "created_at": "2026-10-23T08:49:28.623906+00:00", + "created_at": "2026-10-23T09:37:20.288881+00:00", "license": "MIT", "git_history_included": false, "full_suite_verified": false, @@ -7823,8 +7823,8 @@ }, { "path": "tests/cli_contract_capture.py", - "sha256": "e35e3ce9e618b79fbc8c3b1d1be535fee55ecce344c9266b0e3728860e65f540", - "bytes": 2044 + "sha256": "72d918a69bae3c15bc051eb02ce77c913d43abe34d0fc4269535c474c839a60b", + "bytes": 2426 }, { "path": "tests/conftest.py", @@ -10178,8 +10178,8 @@ }, { "path": "tests/test_cli_registration_contract.py", - "sha256": "0a4ccc347ae8e5ca2b5dcbf6c13e629f8babf5ead6f9485332cf4d9e830fee30", - "bytes": 6062 + "sha256": "bcc2ff5ee1f49956e694766b2b9d29b0809c0689fb140c3dfc5cc62c5a154d64", + "bytes": 6216 }, { "path": "tests/test_cli_shell.py", diff --git a/tests/cli_contract_capture.py b/tests/cli_contract_capture.py index 775027f..91a1061 100644 --- a/tests/cli_contract_capture.py +++ b/tests/cli_contract_capture.py @@ -4,13 +4,10 @@ import inspect import json +import os from pathlib import Path from unittest.mock import patch -from typer.testing import CliRunner - -from thoth.apps import workspace_paths - INVOCATIONS = ( ("--help",), ("doctor", "--help"), @@ -29,6 +26,19 @@ def capture() -> dict[str, object]: + environment = dict(os.environ) + for name in ("GITHUB_ACTIONS", "FORCE_COLOR", "PY_COLORS", "TERMINAL_WIDTH"): + environment.pop(name, None) + # Typer reads these flags at import time; normalize before importing the CLI stack. + with patch.dict(os.environ, environment, clear=True): + return _capture() + + +def _capture() -> dict[str, object]: + from typer.testing import CliRunner + + from thoth.apps import workspace_paths + with patch.object( workspace_paths, "default_workspace", return_value=Path("contract-workspace") ): diff --git a/tests/test_cli_registration_contract.py b/tests/test_cli_registration_contract.py index b18d5f7..1affc8a 100644 --- a/tests/test_cli_registration_contract.py +++ b/tests/test_cli_registration_contract.py @@ -22,8 +22,11 @@ def capture_environment() -> dict[str, str]: + environment = dict(os.environ) + for name in ("GITHUB_ACTIONS", "FORCE_COLOR", "PY_COLORS", "TERMINAL_WIDTH"): + environment.pop(name, None) return { - **os.environ, + **environment, "COLUMNS": "120", "TERM": "dumb", "NO_COLOR": "1", From 804c97afb563fa8c207cf26c8a914741284f819b Mon Sep 17 00:00:00 2001 From: THOTH Date: Fri, 23 Oct 2026 20:48:42 +0900 Subject: [PATCH 3/3] Fix CI-only test failures, use sixteen integration shards, add grounding replay tests Scripted research models answer the v3 contract and table filler role; the CLI help test ignores CI terminal flags; integration tests run in sixteen shards; model-free prediction grounding replay data and scorer added. --- .github/workflows/product-check.yml | 6 +- SOURCE_MANIFEST.json | 77 +- docs/architecture/verification-profiles.md | 2 +- scripts/ci_test_shards.py | 6 +- scripts/ci_test_summary.py | 2 +- scripts/hypothesis_grounding_replay.py | 311 +++++ tests/ci/test_ci_test_shards.py | 16 +- tests/ci/test_ci_test_summary.py | 22 +- tests/fixtures/hypothesis_grounding/README.md | 43 + .../hypothesis_grounding/baseline.json | 173 +++ .../fixtures/hypothesis_grounding/cases.json | 1175 +++++++++++++++++ .../fixtures/scripted_hypothesis_responses.py | 110 ++ .../test_a02_autonomous_acquisition.py | 21 +- .../test_current_hero_normal_entry.py | 24 +- .../test_four_projectpack_portability.py | 27 +- tests/integration/test_research_request_v2.py | 9 +- .../unit/test_hypothesis_grounding_replay.py | 254 ++++ .../test_scripted_hypothesis_responses.py | 175 +++ 18 files changed, 2404 insertions(+), 49 deletions(-) create mode 100644 scripts/hypothesis_grounding_replay.py create mode 100644 tests/fixtures/hypothesis_grounding/README.md create mode 100644 tests/fixtures/hypothesis_grounding/baseline.json create mode 100644 tests/fixtures/hypothesis_grounding/cases.json create mode 100644 tests/fixtures/scripted_hypothesis_responses.py create mode 100644 tests/unit/test_hypothesis_grounding_replay.py create mode 100644 tests/unit/test_scripted_hypothesis_responses.py diff --git a/.github/workflows/product-check.yml b/.github/workflows/product-check.yml index ee4864c..f0d12ef 100644 --- a/.github/workflows/product-check.yml +++ b/.github/workflows/product-check.yml @@ -137,7 +137,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7] + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] defaults: run: shell: pwsh @@ -169,7 +169,7 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Validate and run integration shard run: | - $thothFilesJson = & .\.venv\Scripts\python.exe scripts/ci_test_shards.py select --count 8 --index ${{ matrix.shard }} + $thothFilesJson = & .\.venv\Scripts\python.exe scripts/ci_test_shards.py select --count 16 --index ${{ matrix.shard }} if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $thothFiles = @($thothFilesJson | ConvertFrom-Json) if ($thothFiles.Count -eq 0) { throw 'Integration shard is empty.' } @@ -207,6 +207,6 @@ jobs: FAST_RESULT: ${{ needs.python-fast.result }} INTEGRATION_RESULT: ${{ needs.python-integration.result }} run: >- - python scripts/ci_test_summary.py --reports test-results/aggregate --shards 8 + python scripts/ci_test_summary.py --reports test-results/aggregate --shards 16 --job-results "$env:FAST_RESULT" "$env:INTEGRATION_RESULT" --markdown-output "$env:GITHUB_STEP_SUMMARY" diff --git a/SOURCE_MANIFEST.json b/SOURCE_MANIFEST.json index d72c4f3..057d23c 100644 --- a/SOURCE_MANIFEST.json +++ b/SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schema_version": "thoth-source-preview-manifest-v1", - "created_at": "2026-10-23T09:37:20.288881+00:00", + "created_at": "2026-10-23T11:45:06.957851+00:00", "license": "MIT", "git_history_included": false, "full_suite_verified": false, @@ -18,8 +18,8 @@ }, { "path": ".github/workflows/product-check.yml", - "sha256": "418474e27a6faf3dda070566a987a69a9b8b5f995f94ca83e0d376012cd0c717", - "bytes": 8122 + "sha256": "5af2bc4e756190e35cec3cc6a725302063b18a512b83876bdfcb006dbe50e9ec", + "bytes": 8154 }, { "path": ".gitignore", @@ -2108,8 +2108,8 @@ }, { "path": "docs/architecture/verification-profiles.md", - "sha256": "edbd7848300938086f040249f2aa88c811506509cde55cc1f5508a3b0628f466", - "bytes": 4307 + "sha256": "0f9816439d8a2a0b220eb249bbe972c44b3049ffd66c4f57987d3111b74c2a08", + "bytes": 4309 }, { "path": "docs/brand/README.md", @@ -3058,13 +3058,13 @@ }, { "path": "scripts/ci_test_shards.py", - "sha256": "dd47bf53bc54e90e5dc91d192d220402376955326f88f394232118eb6f8c7a3c", - "bytes": 7452 + "sha256": "4a97a87737246f832a5daeab22f4308c366313c5a24c1975fc218bb83ea7af9c", + "bytes": 7455 }, { "path": "scripts/ci_test_summary.py", - "sha256": "408c84601430e119f2a0ad2d079903b9d35ba5b79a42e75e4c266d7bd0f2f466", - "bytes": 2950 + "sha256": "faffb885f3b210cf05ac96146e642b8efe59aabc06ccbd398c08ec83327a38ea", + "bytes": 2951 }, { "path": "scripts/complete_architecture_gate.py", @@ -3091,6 +3091,11 @@ "sha256": "3b7212b126c2a076c6b4cea8db292617db6b32eae9ddeca22a6d184031d3b007", "bytes": 8909 }, + { + "path": "scripts/hypothesis_grounding_replay.py", + "sha256": "94d6603181566c2eb47d9fc6000153bf7f312ddcbd532f8c8c326781403a489b", + "bytes": 12496 + }, { "path": "scripts/hypothesis_v3_metrics.py", "sha256": "a460e75ebda654e475c355f98f099092ee650caa8fdaa5cab2f693997cf9220f", @@ -7813,13 +7818,13 @@ }, { "path": "tests/ci/test_ci_test_shards.py", - "sha256": "8233774edbf652f283e837d1d6bc816d3895a2ff40256f7fa1983ee55fba0970", - "bytes": 5139 + "sha256": "6835029b914a67030daacd423fb16a4c2a56371876c5a34789b4f6bf3c55f0e5", + "bytes": 5318 }, { "path": "tests/ci/test_ci_test_summary.py", - "sha256": "ba729abb8d3dc01f52853ca118e7665c58985790329b4ea16c488b991aea6fda", - "bytes": 2561 + "sha256": "4e477fceb7a32c67d31e8200ce9e9ca7619c1d848a113c747e544ced868e90f1", + "bytes": 2907 }, { "path": "tests/cli_contract_capture.py", @@ -7961,6 +7966,21 @@ "sha256": "498dcb6cfad2279b1f8a76b8c4cf14910b81ba221768ba5a5ce19ffa8aa1c195", "bytes": 35832 }, + { + "path": "tests/fixtures/hypothesis_grounding/README.md", + "sha256": "4cf8e0c164bcfa2c5258bd4d06bf01298fa991c24ee9ff37e6c14cd4273857be", + "bytes": 2844 + }, + { + "path": "tests/fixtures/hypothesis_grounding/baseline.json", + "sha256": "04cec957721e59404608b4d39717f8f118c978edd9855113d6df4c4acfe1f490", + "bytes": 8709 + }, + { + "path": "tests/fixtures/hypothesis_grounding/cases.json", + "sha256": "2bc6b8bb2ccf1c33e506790eb3cec7fe4e0209a93bbca8476409975b4857d80e", + "bytes": 41592 + }, { "path": "tests/fixtures/hypothesis_input_schemas.json", "sha256": "6448a35592adffb56c088f5d1cff16653be0d0e68ae08cf95de44d2050575697", @@ -8091,6 +8111,11 @@ "sha256": "18b4236f9f0d283985bbfc15b170d9c62bfaa328248ea0f7974bbc76b9bf39d3", "bytes": 112407 }, + { + "path": "tests/fixtures/scripted_hypothesis_responses.py", + "sha256": "403e7fee0186f57e99af7ed7edce792810f3c761fe9a06870e5ced001b745948", + "bytes": 4437 + }, { "path": "tests/fixtures/sealable_methods.json", "sha256": "feba3ef6fcde2e83ba7b5c9c376573a94b02dafaeafcf8da6ce4cefef14efb67", @@ -8308,8 +8333,8 @@ }, { "path": "tests/integration/test_a02_autonomous_acquisition.py", - "sha256": "1b51c5a9e3f9e2372551ce7f9f3a25c0d2366f62295197c9382c31eabc7689aa", - "bytes": 24619 + "sha256": "0b394d20691fadf60a056e431e9bf4a5538859f6224312ce4dd0a3a127ad7438", + "bytes": 25120 }, { "path": "tests/integration/test_a02_real_connector_routes.py", @@ -8678,8 +8703,8 @@ }, { "path": "tests/integration/test_current_hero_normal_entry.py", - "sha256": "48f26bb509f2576ea79cc17779fd5ebbfbd6d3dcea10d3626e2bcddd3e4fc527", - "bytes": 1718 + "sha256": "8b80e56c10f2a56d6ef70b9904b9a236176eb4ec31d762e541b653e532c4ec0c", + "bytes": 2722 }, { "path": "tests/integration/test_current_secondary_same_trace.py", @@ -8768,8 +8793,8 @@ }, { "path": "tests/integration/test_four_projectpack_portability.py", - "sha256": "12347d341a8c6ef57313df45598f404e570a5cc352f512763c5f101b4e9bc5fd", - "bytes": 11888 + "sha256": "ba257a035b5339ed1bcd330a413fc66477c86abe35c19f504278e031f2cfea8c", + "bytes": 12960 }, { "path": "tests/integration/test_full_backend_adversarial_isolation.py", @@ -9623,8 +9648,8 @@ }, { "path": "tests/integration/test_research_request_v2.py", - "sha256": "d1400bc76e15f9a3bcc5cc764e3f4020b35e30b6d704d2aa449f6fc30137ae5b", - "bytes": 20993 + "sha256": "ff6594cbc1b13352a105d14289e007fd03c6c5082e0186953d93068012c51aa5", + "bytes": 21334 }, { "path": "tests/integration/test_research_review_order.py", @@ -11346,6 +11371,11 @@ "sha256": "ab85b5c6741437de4c3f9b14b88d8ca620985c46562eba2a4ee1bfcb3b811a3f", "bytes": 3142 }, + { + "path": "tests/unit/test_hypothesis_grounding_replay.py", + "sha256": "0d560aa77710f47b71d8a0bdf11e3c948b63f861ab78116eef658091ac81f7f7", + "bytes": 10806 + }, { "path": "tests/unit/test_hypothesis_v3_metrics.py", "sha256": "88e55392895bb63a1da1ad7c26abbee88778d813194bd29779ecb51dea18020f", @@ -11551,6 +11581,11 @@ "sha256": "748c245fba5caeea53f783e245da8c21c3348e62809664637583554677d01c5e", "bytes": 3596 }, + { + "path": "tests/unit/test_scripted_hypothesis_responses.py", + "sha256": "bdee711069a62d4014704a31f50bbf1122c57c444110d26846a0f24b22857877", + "bytes": 6726 + }, { "path": "tests/unit/test_selected_evidence_reader.py", "sha256": "6c0f04e6b38768fd8627ab06943f6c98d8732f1b79393437aa6f39bd386fb016", diff --git a/docs/architecture/verification-profiles.md b/docs/architecture/verification-profiles.md index 83276fb..f22ed7b 100644 --- a/docs/architecture/verification-profiles.md +++ b/docs/architecture/verification-profiles.md @@ -7,7 +7,7 @@ During development, retain the focused checks specified in AGENTS.md. The Product check workflow provides this check on clean Windows runners for pull requests, main pushes, manual dispatches and the nightly default-branch run (18:00 UTC / 03:00 KST). The existing `checks` job keeps the static, Web and quick-start checks. `python-fast` collects -everything except `tests/integration`; eight `python-integration` matrix jobs cover every +everything except `tests/integration`; sixteen `python-integration` matrix jobs cover every integration test file exactly once. Each uses `-q -rfE -p no:cacheprovider` plus a JUnit report, with the same root conftest and inactive-Hook selection policy as a local full run. diff --git a/scripts/ci_test_shards.py b/scripts/ci_test_shards.py index 9bd4525..9464e84 100644 --- a/scripts/ci_test_shards.py +++ b/scripts/ci_test_shards.py @@ -3,8 +3,8 @@ Refresh timings from a complete JUnit report (including failed tests): python scripts/ci_test_shards.py refresh --junit test-results/python.xml Preview all groups or select the JSON file list for one zero-based group: - python scripts/ci_test_shards.py plan --count 8 - python scripts/ci_test_shards.py select --count 8 --index 0 + python scripts/ci_test_shards.py plan --count 16 + python scripts/ci_test_shards.py select --count 16 --index 0 Only repository-relative integration paths and elapsed seconds are persisted. New files use the median positive measured duration (one second if none exist), then join the lightest group. @@ -129,7 +129,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("command", choices=("plan", "select", "refresh")) parser.add_argument("--root", type=Path, default=ROOT) parser.add_argument("--durations", type=Path, default=TIMINGS) - parser.add_argument("--count", type=int, default=8) + parser.add_argument("--count", type=int, default=16) parser.add_argument("--index", type=int) parser.add_argument("--junit", type=Path) args = parser.parse_args(argv) diff --git a/scripts/ci_test_summary.py b/scripts/ci_test_summary.py index 8113f43..373e68d 100644 --- a/scripts/ci_test_summary.py +++ b/scripts/ci_test_summary.py @@ -53,7 +53,7 @@ def summarize(directory: Path, shards: int, job_results: list[str]) -> tuple[str def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--reports", type=Path, required=True) - parser.add_argument("--shards", type=int, default=8) + parser.add_argument("--shards", type=int, default=16) parser.add_argument("--job-results", nargs=2, required=True) parser.add_argument("--markdown-output", type=Path) args = parser.parse_args() diff --git a/scripts/hypothesis_grounding_replay.py b/scripts/hypothesis_grounding_replay.py new file mode 100644 index 0000000..fb4e82a --- /dev/null +++ b/scripts/hypothesis_grounding_replay.py @@ -0,0 +1,311 @@ +"""Offline grading of labelled grounding cases, not a production grounding implementation. + +The scorer checks supplied structured facts and an independently labelled context. It neither +infers natural-language entailment nor creates links, changes records, or calls a model. +Usage: python scripts/hypothesis_grounding_replay.py check-fixtures + python scripts/hypothesis_grounding_replay.py grade --predictions predictions.json +Predictions must be a JSON object from case ID to Proposal (see the fixture README). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from decimal import Decimal +from pathlib import Path +from typing import Literal, cast + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator + +Role = Literal["OBSERVATION_RESTATEMENT", "EVIDENCE_INFERENCE", "HYPOTHESIS_ONLY", "UNASSESSED"] +Origin = Literal["SYNTHETIC", "MEASURED", "UNASSESSED"] +FIXTURES = Path(__file__).resolve().parents[1] / "tests/fixtures/hypothesis_grounding" + + +class FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class Binding(FrozenModel): + span_id: str = Field(min_length=1) + source_version_id: str = Field(min_length=1) + text_sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + +class Observation(FrozenModel): + result_id: str + subject_id: str + condition: dict[str, str] + numerator: int = Field(ge=0, strict=True) + denominator: int = Field(gt=0, strict=True) + value: Decimal + unit: str + observed_at: AwareDatetime + synthetic: bool = Field(strict=True) + + @field_validator("value") + @classmethod + def finite_value(cls, value: Decimal) -> Decimal: + if not value.is_finite(): + raise ValueError("observation value must be finite") + return value + + +class Span(FrozenModel): + binding: Binding + project_id: str + observation: Observation + exact_text: str + relation: Literal["SUPPORT", "COUNTER", "REFERENCE_ONLY"] + + +class ReplayContext(FrozenModel): + kind: Literal["OBSERVATION", "INFERENCE", "FUTURE", "LEGACY"] + project_id: str + cutoff_at: AwareDatetime + data_origin: Origin + target: Observation | None = None + evidence: tuple[Span, ...] = () + permitted_span_ids: tuple[str, ...] = () + pinned_sources: tuple[Binding, ...] = () + legacy_basis: tuple[Binding, ...] = () + legacy_knowledge_state: Literal["PREDICTED", "UNKNOWN"] | None = None + legacy_expected: str | None = None + # Independently labelled supporting premises, never chosen by the proposal being graded. + inference_support_ids: tuple[str, ...] = () + + @model_validator(mode="after") + def unambiguous_context(self) -> ReplayContext: + for identities in ( + [span.binding.span_id for span in self.evidence], + [binding.span_id for binding in self.pinned_sources], + [binding.span_id for binding in self.legacy_basis], + list(self.permitted_span_ids), + list(self.inference_support_ids), + ): + if len(set(identities)) != len(identities): + raise ValueError("duplicate context identity") + if self.kind == "OBSERVATION" and self.target is None: + raise ValueError("observation context requires a target") + if self.kind == "LEGACY" and ( + self.legacy_knowledge_state is None + or (self.legacy_knowledge_state == "UNKNOWN") != (self.legacy_expected is None) + ): + raise ValueError("legacy context requires the original prediction state and text") + return self + + +class Proposal(FrozenModel): + """Replay assessment envelope; bindings/verification_claim are not model-granted authority.""" + + schema_version: Literal["grounding_replay_proposal.v1"] = "grounding_replay_proposal.v1" + role: Role + knowledge_state: Literal["PREDICTED", "UNKNOWN"] + expected: str | None = Field(default=None, min_length=1, max_length=80) + rationale: str = Field(default="", max_length=500) + basis: tuple[Binding, ...] = Field(default=(), max_length=5) + observation: Observation | None = None + data_origin: Origin + verification_claim: bool = Field(default=False, strict=True) + + @model_validator(mode="after") + def valid_prediction_state(self) -> Proposal: + if (self.knowledge_state == "UNKNOWN") != (self.expected is None): + raise ValueError("prediction state conflicts with expected text") + if self.expected is not None and not self.expected.strip(): + raise ValueError("empty prediction text") + return self + + +class Score(FrozenModel): + passed: bool + reasons: tuple[str, ...] + false_verification_promotions: int + unauthorized_links: int + invalid_links: int + missing_observation_links: int + + +def link_problems(context: ReplayContext, binding: Binding) -> tuple[str, ...]: + spans = {span.binding.span_id: span for span in context.evidence} + pinned = {item.span_id: item for item in context.pinned_sources} + problems: list[str] = [] + span = spans.get(binding.span_id) + if binding.span_id not in context.permitted_span_ids: + problems.append("UNAUTHORIZED_LINK") + if span is None: + problems.append("UNKNOWN_SPAN") + else: + if span.project_id != context.project_id: + problems.append("FOREIGN_PROJECT_LINK") + if binding != span.binding or binding != pinned.get(binding.span_id): + problems.append("PINNED_SOURCE_MISMATCH") + if hashlib.sha256(span.exact_text.encode("utf-8")).hexdigest() != binding.text_sha256: + problems.append("TEXT_HASH_MISMATCH") + if span.observation.observed_at > context.cutoff_at: + problems.append("AFTER_CUTOFF") + if context.kind != "LEGACY" and span.relation != "SUPPORT": + problems.append("NON_SUPPORTING_EVIDENCE") + return tuple(problems) + + +def score_row(context: ReplayContext, proposal: Proposal) -> Score: + """Pure comparison with labelled facts. A passing proposal remains unverified.""" + reasons: list[str] = [] + unauthorized = invalid = 0 + for binding in proposal.basis: + problems = link_problems(context, binding) + reasons.extend(problems) + invalid += bool(problems) + unauthorized += any(p in problems for p in ("UNAUTHORIZED_LINK", "FOREIGN_PROJECT_LINK")) + ids = {binding.span_id for binding in proposal.basis} + if len(ids) != len(proposal.basis): + reasons.append("DUPLICATE_LINK") + promotions = int(proposal.verification_claim) + if proposal.verification_claim: + reasons.append("UNAUTHORIZED_VERIFICATION_PROMOTION") + if proposal.data_origin != context.data_origin: + reasons.append("DATA_ORIGIN_MISMATCH") + if proposal.data_origin == "MEASURED": + promotions = 1 + if proposal.role != "UNASSESSED" and not proposal.rationale.strip(): + reasons.append("GROUNDING_RATIONALE_REQUIRED") + eligible: set[str] = ( + { + span.binding.span_id + for span in context.evidence + if span.observation == context.target and not link_problems(context, span.binding) + } + if context.kind == "OBSERVATION" + else set() + ) + missing = int(bool(eligible) and not ids) + if eligible: + if proposal.role != "OBSERVATION_RESTATEMENT": + reasons.append("OBSERVATION_ROLE_REQUIRED") + if not ids: + reasons.append("MISSING_REQUIRED_OBSERVATION_LINK") + elif not ids <= eligible: + reasons.append("OBSERVATION_LINK_MISMATCH") + if proposal.observation != context.target or proposal.knowledge_state != "PREDICTED": + reasons.append("OBSERVATION_CONTENT_MISMATCH") + elif context.kind == "INFERENCE": + if proposal.role != "EVIDENCE_INFERENCE": + reasons.append("INFERENCE_ROLE_REQUIRED") + if not ids or not ids <= set(context.inference_support_ids): + reasons.append("INFERENCE_PREMISE_MISMATCH") + if proposal.observation is not None: + reasons.append("INFERENCE_IS_NOT_AN_OBSERVATION") + if proposal.knowledge_state != "PREDICTED": + reasons.append("INFERENCE_PREDICTION_REQUIRED") + elif context.kind == "FUTURE": + role = "HYPOTHESIS_ONLY" if proposal.knowledge_state == "PREDICTED" else "UNASSESSED" + if proposal.role != role or ids or proposal.observation is not None: + reasons.append("UNSUPPORTED_FUTURE_GROUNDING") + elif context.kind == "LEGACY": + if proposal.role != "UNASSESSED" or proposal.observation is not None: + reasons.append("LEGACY_ROLE_MUST_REMAIN_UNASSESSED") + if proposal.basis != context.legacy_basis: + reasons.append("LEGACY_BASIS_CHANGED") + if (proposal.knowledge_state, proposal.expected) != ( + context.legacy_knowledge_state, + context.legacy_expected, + ): + reasons.append("LEGACY_PREDICTION_CHANGED") + else: + if proposal.role != "UNASSESSED" or ids or proposal.observation is not None: + reasons.append("UNASSESSED_REQUIRED") + return Score( + passed=not reasons, + reasons=tuple(sorted(set(reasons))), + false_verification_promotions=promotions, + unauthorized_links=unauthorized, + invalid_links=invalid, + missing_observation_links=missing, + ) + + +class Example(FrozenModel): + case_id: str + category: str + context: ReplayContext + accepted: Proposal + rejected: Proposal + + +class ReplaySuite(FrozenModel): + schema_version: Literal["grounding_replay_cases.v1"] + examples: tuple[Example, ...] = Field(min_length=1) + + +def _unique_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("check-fixtures", "grade")) + parser.add_argument("--cases", type=Path, default=FIXTURES / "cases.json") + parser.add_argument("--predictions", type=Path) + args = parser.parse_args(argv) + try: + source = args.cases.read_bytes() + suite = ReplaySuite.model_validate_json(source) + except (OSError, ValueError): + parser.error("invalid or unreadable replay case suite") + if len({case.case_id for case in suite.examples}) != len(suite.examples): + parser.error("duplicate case ID") + results: dict[str, object] = {} + if args.command == "check-fixtures": + passed = True + for case in suite.examples: + good = score_row(case.context, case.accepted) + bad = score_row(case.context, case.rejected) + passed = passed and good.passed and not bad.passed + results[case.case_id] = {"accepted": good.model_dump(), "rejected": bad.model_dump()} + else: + if args.predictions is None: + parser.error("grade requires --predictions") + try: + raw = json.loads( + args.predictions.read_text(encoding="utf-8"), object_pairs_hook=_unique_pairs + ) + except (OSError, ValueError): + parser.error("invalid, duplicate-key, or unreadable predictions") + if not isinstance(raw, dict): + parser.error("predictions must be an object") + predictions = cast(dict[str, object], raw) + if set(predictions) != {case.case_id for case in suite.examples}: + parser.error("predictions must cover every case exactly once") + passed = True + for case in suite.examples: + try: + proposal = Proposal.model_validate(predictions[case.case_id]) + except ValueError: + parser.error(f"invalid proposal for {case.case_id}") + result = score_row(case.context, proposal) + passed = passed and result.passed + results[case.case_id] = result.model_dump() + print( + json.dumps( + { + "scope": "OFFLINE_LABELLED_REPLAY", + "case_set_sha256": hashlib.sha256(source).hexdigest(), + "passed": passed, + "results": results, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ci/test_ci_test_shards.py b/tests/ci/test_ci_test_shards.py index 996b476..27f0dc7 100644 --- a/tests/ci/test_ci_test_shards.py +++ b/tests/ci/test_ci_test_shards.py @@ -43,9 +43,9 @@ def test_new_file_joins_lightest_group_and_removed_timing_is_ignored() -> None: def test_all_new_files_have_a_positive_balancing_estimate() -> None: - files = [f"tests/integration/test_{i}.py" for i in range(8)] - groups, loads = build_shards(files, {}, 8) - assert loads == [1.0] * 8 and all(len(group) == 1 for group in groups) + files = [f"tests/integration/test_{i}.py" for i in range(16)] + groups, loads = build_shards(files, {}, 16) + assert loads == [1.0] * 16 and all(len(group) == 1 for group in groups) @pytest.mark.parametrize("groups", [[["a"], []], [["a"], ["a", "b"]], [["a", "b", "c"]]]) @@ -129,9 +129,15 @@ def test_select_revalidates_actual_files_and_refuses_empty_shard( assert error.value.code == 2 -def test_real_repository_partition_is_complete_and_disjoint() -> None: +def test_real_repository_partition_is_complete_and_disjoint( + capsys: pytest.CaptureFixture[str], +) -> None: root = Path(__file__).resolve().parents[2] files = discover(root) - groups, _ = build_shards(files, load_durations(root / "tests/ci/integration_durations.json"), 8) + groups, _ = build_shards( + files, load_durations(root / "tests/ci/integration_durations.json"), 16 + ) validate_partition(files, groups) assert sum(map(len, groups)) == len(files) + assert main(["plan", "--root", str(root)]) == 0 + assert len(json.loads(capsys.readouterr().out)["shards"]) == 16 diff --git a/tests/ci/test_ci_test_summary.py b/tests/ci/test_ci_test_summary.py index 2ba333b..7c802ce 100644 --- a/tests/ci/test_ci_test_summary.py +++ b/tests/ci/test_ci_test_summary.py @@ -1,9 +1,10 @@ """Missing artifacts, failed jobs and duplicate tests cannot produce a passing summary.""" +import sys from pathlib import Path import pytest -from scripts.ci_test_summary import summarize +from scripts.ci_test_summary import main, summarize def write_report(path: Path, name: str, detail: str = "") -> None: @@ -14,13 +15,22 @@ def write_report(path: Path, name: str, detail: str = "") -> None: ) -def test_summary_counts_every_report_and_keeps_skips_visible(tmp_path: Path) -> None: +def test_summary_counts_every_report_and_keeps_skips_visible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: write_report(tmp_path / "python-fast.xml", "fast", "") - for i in range(8): + for i in range(16): write_report(tmp_path / f"python-integration-{i}.xml", f"integration-{i}") - report, passed = summarize(tmp_path, 8, ["success", "success"]) - assert passed and "| **Total** | 9 | 0 | 0 | 1 |" in report - assert all(f"python-integration-{i}" in report for i in range(8)) + report, passed = summarize(tmp_path, 16, ["success", "success"]) + assert passed and "| **Total** | 17 | 0 | 0 | 1 |" in report + assert all(f"python-integration-{i}" in report for i in range(16)) + monkeypatch.setattr( + sys, + "argv", + ["ci_test_summary", "--reports", str(tmp_path), "--job-results", "success", "success"], + ) + assert main() == 0 + assert "| **Total** | 17 | 0 | 0 | 1 |" in capsys.readouterr().out @pytest.mark.parametrize("status", ["failure", "cancelled", "skipped"]) diff --git a/tests/fixtures/hypothesis_grounding/README.md b/tests/fixtures/hypothesis_grounding/README.md new file mode 100644 index 0000000..efff563 --- /dev/null +++ b/tests/fixtures/hypothesis_grounding/README.md @@ -0,0 +1,43 @@ +# Offline prediction-grounding replay data + +These files are test inputs, not production records, model instructions or evidence of a real +experiment. They require no server, account, network request or local private directory. + +`baseline.json` preserves a recorded synthetic result's four hypothesis statements, five tests, +twenty expected-result rows, empty basis lists, skipped table filling and zero dropped rows. +The source-byte digest records provenance; the payload digest seals this public projection. +The tests pin both the payload digest and the case-file digest. Changing a baseline requires +an explicit reviewed change, not automatic resealing when a test fails. + +`cases.json` contains two positive controls and ten adverse categories. Each independently +labelled context contains structured observations, permitted span IDs and pinned source +identities. It has an accepted output and a rejected output so a grader that always passes or +always fails cannot satisfy the controls. The categories are numeric coincidence, condition, +unit, denominator, pinned revision, scope, invented span, future effect, counterevidence and +synthetic data presented as measurement. + +The four role names are `OBSERVATION_RESTATEMENT`, `EVIDENCE_INFERENCE`, `HYPOTHESIS_ONLY` and +`UNASSESSED`. They do not confer verification authority. A forecast can have no direct supporting +observation and still be a valid unconfirmed hypothesis-only prediction. Old rows retain their +unassessed role and original links; empty links do not justify retroactive classification. + +Run the controls or grade a separate candidate file: + +```text +python scripts/hypothesis_grounding_replay.py check-fixtures +python scripts/hypothesis_grounding_replay.py grade --predictions predictions.json +python -m pytest tests/unit/test_hypothesis_grounding_replay.py +``` + +The candidate file is a JSON object keyed by **every** case ID. Each value follows the accepted +example's `Proposal` structure (`grounding_replay_proposal.v1`). Its bound references and +verification claim describe the output being evaluated; they do not let that output choose +its own permitted evidence or truth labels. Missing cases fail input validation. Grading returns +exit 0 only when every row passes, exit 1 for rejected output, and exit 2 for invalid CLI input. + +The scorer compares structured facts and reference identities. It does not understand arbitrary +natural-language entailment, attach evidence, alter a database, validate causal effects or +implement a new product schema. Product integration and model-quality evaluation remain separate. +The assessment envelope's observation, origin and verification labels must come from an +independent interpretation of the displayed output. Feeding a model's unverified self-description +as those labels would not test whether its prose makes a false claim. diff --git a/tests/fixtures/hypothesis_grounding/baseline.json b/tests/fixtures/hypothesis_grounding/baseline.json new file mode 100644 index 0000000..2067900 --- /dev/null +++ b/tests/fixtures/hypothesis_grounding/baseline.json @@ -0,0 +1,173 @@ +{ + "schema_version": "grounding_replay_baseline.v1", + "source_kind": "RECORDED_SYNTHETIC_RESEARCH", + "source_sha256": "59d3042a31b1059bfc1f5a239742a2606f606e29f908bf78c6c551ecd5d11868", + "payload_sha256": "6fffc576d483e4c64ad5ed1b4cd9ea51add280f4b38ccef985405faeeca9ea3b", + "payload": { + "hypotheses": [ + { + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1", + "statement": "건조와 우천의 차이는 강우 자체보다 표적·거리·설정 차이 또는 작은 표본의 변동에서 생겼을 수 있다." + }, + { + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2", + "statement": "강우 경로에서 표적 신호가 약해져 탐지에 필요한 신호를 확보하지 못했을 수 있다." + }, + { + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3", + "statement": "센서 표면의 물막이 표적 신호를 약하게 만들어 탐지율이 낮아졌을 수 있다." + }, + { + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4", + "statement": "표적 신호 자체의 손실 없이도, 강우 반사를 처리하는 탐지·추적 설정이 맞지 않아 표적을 누락했을 수 있다." + } + ], + "tests": [ + { + "designed_for": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1", + "test_id": "test:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:matched", + "procedure_candidate": "시험 담당자 승인 후 같은 표적·거리·설정·교정 상태에서 건조와 우천 순서를 무작위로 바꾸어 반복한다. 우천의 경로 강우와 표면 젖음은 따로 기록한다. 구분할 최소 차이와 반복 수를 미리 정하고 탐지율 차이를 비교한다.", + "rows": [ + { + "basis": [], + "expected": "우천 저하가 재현되지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1" + }, + { + "basis": [], + "expected": "우천 저하가 재현됨", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2" + }, + { + "basis": [], + "expected": "우천 저하가 재현됨", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3" + }, + { + "basis": [], + "expected": "우천 저하가 재현됨", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4" + } + ] + }, + { + "designed_for": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1", + "test_id": "test:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:signal", + "procedure_candidate": "시험 담당자 승인 후 표적·거리·수신 이득·교정 상태를 고정하고 건조와 우천을 반복 비교한다. 탐지 처리 전의 표적 신호 세기를 정답 표적에 맞춰 측정한다. 배경 반사와 표면 상태도 기록하여 표적 신호 손실과 처리 단계 누락을 구분한다.", + "rows": [ + { + "basis": [], + "expected": "처리 전 표적 신호 손실이 재현되지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1" + }, + { + "basis": [], + "expected": "처리 전 표적 신호 손실이 재현됨", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2" + }, + { + "basis": [], + "expected": "처리 전 표적 신호 손실이 재현됨", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3" + }, + { + "basis": [], + "expected": "처리 전 표적 신호 손실이 재현되지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4" + } + ] + }, + { + "designed_for": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2", + "test_id": "test:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:path", + "procedure_candidate": "시험 담당자 승인과 안전성 확인 후 센서 표면은 건조하게 유지한다. 표적·거리·수신 이득·처리 설정을 고정하고 경로 강우만 단계별로 바꿔 반복한다. 처리 전 표적 신호, 배경 반사, 탐지율을 함께 측정한다.", + "rows": [ + { + "basis": [], + "expected": "경로 강우에 따른 신호 손실과 탐지 저하의 동반 변화 없음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1" + }, + { + "basis": [], + "expected": "경로 강우에 따른 신호 손실과 탐지 저하의 동반 변화 있음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2" + }, + { + "basis": [], + "expected": "경로 강우에 따른 신호 손실과 탐지 저하의 동반 변화 없음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3" + }, + { + "basis": [], + "expected": "경로 강우에 따른 신호 손실과 탐지 저하의 동반 변화 없음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4" + } + ] + }, + { + "designed_for": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3", + "test_id": "test:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:surface", + "procedure_candidate": "시험 담당자 승인과 장비의 젖음 허용 범위 확인 후 경로 강우·표적·거리·설정을 고정한다. 표면 건조·젖음·재건조를 반복 비교하고 처리 전 표적 신호와 탐지율을 기록한다.", + "rows": [ + { + "basis": [], + "expected": "표면 상태에 따른 저하와 회복이 재현되지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1" + }, + { + "basis": [], + "expected": "표면 상태에 따른 저하와 회복이 재현되지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2" + }, + { + "basis": [], + "expected": "표면 상태에 따른 저하와 회복이 재현됨", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3" + }, + { + "basis": [], + "expected": "표면 상태에 따른 저하와 회복이 재현되지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4" + } + ] + }, + { + "designed_for": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4", + "test_id": "test:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:replay", + "procedure_candidate": "격리된 재처리 환경 담당자 승인 후 조건을 맞춘 반복 시험의 건조·우천 원시 입력을 복사해 사용한다. 표적 있음·없음 입력을 모두 포함한다. 원본과 버전을 보존하고 관련 설정을 하나씩 바꾼다. 처리 전 표적 신호, 단계별 누락, 탐지율, 허위 추적을 함께 비교한다.", + "rows": [ + { + "basis": [], + "expected": "신호 유지·처리 누락·설정 수정 후 회복이 함께 나타나지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:1" + }, + { + "basis": [], + "expected": "신호 유지·처리 누락·설정 수정 후 회복이 함께 나타나지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:2" + }, + { + "basis": [], + "expected": "신호 유지·처리 누락·설정 수정 후 회복이 함께 나타나지 않음", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:3" + }, + { + "basis": [], + "expected": "신호 유지·처리 누락·설정 수정 후 회복이 함께 나타남", + "hypothesis_id": "hypothesis:737f13e4-bb13-45ab-9a26-1c9f72666661:rain:4" + } + ] + } + ], + "table_fill": { + "answers": [], + "cells": 20, + "changed_cells": 0, + "open_after": 0, + "open_before": 0, + "reason_code": null, + "state": "SKIPPED_FULL" + }, + "dropped_expected_results": [] + } +} diff --git a/tests/fixtures/hypothesis_grounding/cases.json b/tests/fixtures/hypothesis_grounding/cases.json new file mode 100644 index 0000000..b57478c --- /dev/null +++ b/tests/fixtures/hypothesis_grounding/cases.json @@ -0,0 +1,1175 @@ +{ + "schema_version": "grounding_replay_cases.v1", + "examples": [ + { + "case_id": "exact-rain-observation", + "category": "POSITIVE_EXACT", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "labelled-inference", + "category": "POSITIVE_INFERENCE", + "context": { + "kind": "INFERENCE", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": null, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [ + "span:rain" + ] + }, + "accepted": { + "role": "EVIDENCE_INFERENCE", + "knowledge_state": "PREDICTED", + "expected": "비 조건의 낮은 기록은 추가 원인 조사를 뒷받침한다", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "EVIDENCE_INFERENCE", + "knowledge_state": "PREDICTED", + "expected": "비 조건의 낮은 기록은 추가 원인 조사를 뒷받침한다", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": true + } + }, + { + "case_id": "number-only", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "7bf39a33f60c6ce242eb5f52d54c6266eefcd279560b9eb093b5c7cabd201e23" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:other", + "subject_id": "criterion:other", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:other\",\"subject_id\":\"criterion:other\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "7bf39a33f60c6ce242eb5f52d54c6266eefcd279560b9eb093b5c7cabd201e23" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "7bf39a33f60c6ce242eb5f52d54c6266eefcd279560b9eb093b5c7cabd201e23" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "condition-mismatch", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "08c8e7cffd853cf6f6bab447cb60c0f1a04f3fae17dc574b709b715e89fbad4b" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "dry" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"dry\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "08c8e7cffd853cf6f6bab447cb60c0f1a04f3fae17dc574b709b715e89fbad4b" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "08c8e7cffd853cf6f6bab447cb60c0f1a04f3fae17dc574b709b715e89fbad4b" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "unit-mismatch", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "701365cddfc62b8f7db7d45e2566840147654dd972f48d29a21145eae20dc6b9" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "per_min", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"per_min\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "701365cddfc62b8f7db7d45e2566840147654dd972f48d29a21145eae20dc6b9" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "701365cddfc62b8f7db7d45e2566840147654dd972f48d29a21145eae20dc6b9" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "denominator-mismatch", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "ae05cd8b91a3f6102ddfb0a3b2292b4c3800b51b8673cef708342deab432aa9c" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 20, + "denominator": 25, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":25,\"numerator\":20,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "ae05cd8b91a3f6102ddfb0a3b2292b4c3800b51b8673cef708342deab432aa9c" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "ae05cd8b91a3f6102ddfb0a3b2292b4c3800b51b8673cef708342deab432aa9c" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "stale-pinned-revision", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v2", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "foreign-or-private-scope", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:foreign", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "invented-span", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:invented", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "past-observation-to-future-effect", + "category": "NEGATIVE", + "context": { + "kind": "FUTURE", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": null, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "HYPOTHESIS_ONLY", + "knowledge_state": "PREDICTED", + "expected": "표면 물막 가설이 맞다면 건조 후 탐지가 회복될 것이다", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "표면 물막 가설이 맞다면 건조 후 탐지가 회복될 것이다", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "counterevidence-as-support", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "COUNTER" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "UNASSESSED", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "EVIDENCE_INFERENCE", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": null, + "data_origin": "SYNTHETIC", + "verification_claim": false + } + }, + { + "case_id": "synthetic-as-measured", + "category": "NEGATIVE", + "context": { + "kind": "OBSERVATION", + "project_id": "project:fixture", + "cutoff_at": "2026-10-08T00:00:00Z", + "data_origin": "SYNTHETIC", + "target": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "evidence": [ + { + "binding": { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + }, + "project_id": "project:fixture", + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "exact_text": "{\"condition\":{\"weather\":\"rain\"},\"denominator\":20,\"numerator\":16,\"observed_at\":\"2026-10-01T00:00:00Z\",\"result_id\":\"result:rain\",\"subject_id\":\"criterion:detection\",\"synthetic\":true,\"unit\":\"ratio\",\"value\":\"0.80\"}", + "relation": "SUPPORT" + } + ], + "permitted_span_ids": [ + "span:rain" + ], + "pinned_sources": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "inference_support_ids": [] + }, + "accepted": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "SYNTHETIC", + "verification_claim": false + }, + "rejected": { + "role": "OBSERVATION_RESTATEMENT", + "knowledge_state": "PREDICTED", + "expected": "비 조건 탐지 관측: 16/20 = 0.80", + "rationale": "합성 재생 사례의 근거 역할을 평가하며 실험 검증을 주장하지 않는다.", + "basis": [ + { + "span_id": "span:rain", + "source_version_id": "source:rain:v1", + "text_sha256": "a499653355658e71f28c402228f92052101a8fc5991d2e6af3e99c826310117a" + } + ], + "observation": { + "result_id": "result:rain", + "subject_id": "criterion:detection", + "condition": { + "weather": "rain" + }, + "numerator": 16, + "denominator": 20, + "value": "0.80", + "unit": "ratio", + "observed_at": "2026-10-01T00:00:00Z", + "synthetic": true + }, + "data_origin": "MEASURED", + "verification_claim": false + } + } + ] +} diff --git a/tests/fixtures/scripted_hypothesis_responses.py b/tests/fixtures/scripted_hypothesis_responses.py new file mode 100644 index 0000000..8406bea --- /dev/null +++ b/tests/fixtures/scripted_hypothesis_responses.py @@ -0,0 +1,110 @@ +"""Schema-aware synthetic model replies, using only the request's supplied context.""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel + +from thoth.domain.hypothesis import HypothesisPortfolio, HypothesisPortfolioV3 +from thoth.domain.hypothesis_v4 import HypothesisPortfolioV4, HypothesisTableFillV4 +from thoth.domain.model import ModelRequest + + +class _FillTask(BaseModel): + designed_for: str + test_id: str + expected_if_true: str + expected_if_alternative: str + open_hypothesis_ids: tuple[str, ...] + + +class _FillContext(BaseModel): + tests: tuple[_FillTask, ...] + + +def _korean(request: ModelRequest[BaseModel]) -> bool: + language = request.response_language or request.context_pack.research_context.get( + "response_language" + ) + return language == "ko" or ( + language is None and re.search(r"[가-힣]", request.context_pack.problem) is not None + ) + + +def _label(korean: bool, own: bool, original: str) -> str: + if korean: + return "예상 차이가 나타남" if own else "예상 차이가 나타나지 않음" + return original + + +def hypothesis_output[T: BaseModel]( + request: ModelRequest[T], payload: dict[str, object], *, complete: bool = False +) -> T: + """Keep v2 exact; v3/v4 emit own cells and optionally leave alternatives for the filler.""" + if not issubclass(request.output_model, HypothesisPortfolioV3): + return request.output_model.model_validate(payload) + base = HypothesisPortfolio.model_validate(payload) + korean = _korean(request) + typed_states = issubclass(request.output_model, HypothesisPortfolioV4) + hypotheses: list[dict[str, object]] = [] + for owner in base.hypotheses: + tests: list[dict[str, object]] = [] + for test in owner.discriminating_tests: + rows: list[dict[str, object]] = [] + for candidate in base.hypotheses: + own = candidate.hypothesis_id == owner.hypothesis_id + if own or complete: + row: dict[str, object] = { + "hypothesis_id": candidate.hypothesis_id, + "expected": _label( + korean, + own, + test.expected_if_true if own else test.expected_if_alternative, + ), + "basis": [], # A scripted prediction is not an observed result. + } + if typed_states: + row["knowledge_state"] = "PREDICTED" + rows.append(row) + tests.append({**test.model_dump(), "expected_by_hypothesis": rows}) + hypotheses.append( + { + **owner.model_dump(), + "discriminating_tests": tests, + "refutation_conditions": [ + "독립 비교에서 대안 결과가 확인되면 이 후보를 재검토한다." + if korean + else ( + "Reconsider this candidate if an independent check matches the alternative." + ) + ] + if tests + else [], + } + ) + return request.output_model.model_validate({**payload, "hypotheses": hypotheses}) + + +def table_fill_output[T: BaseModel](request: ModelRequest[T]) -> T: + """Reply only to requested open cells, retaining IDs and the requested v3/v4 schema.""" + context = _FillContext.model_validate(request.context_pack.research_context["table_filler"]) + korean = _korean(request) + typed_states = issubclass(request.output_model, HypothesisTableFillV4) + tests: list[dict[str, object]] = [] + for test in context.tests: + rows: list[dict[str, object]] = [] + for hypothesis_id in test.open_hypothesis_ids: + own = hypothesis_id == test.designed_for + row: dict[str, object] = { + "hypothesis_id": hypothesis_id, + "expected": _label( + korean, own, test.expected_if_true if own else test.expected_if_alternative + ), + "basis": [], + } + if typed_states: + row["knowledge_state"] = "PREDICTED" + rows.append(row) + tests.append({"designed_for": test.designed_for, "test_id": test.test_id, "rows": rows}) + return request.output_model.model_validate({"tests": tests}) diff --git a/tests/integration/test_a02_autonomous_acquisition.py b/tests/integration/test_a02_autonomous_acquisition.py index 9b57115..f8664b5 100644 --- a/tests/integration/test_a02_autonomous_acquisition.py +++ b/tests/integration/test_a02_autonomous_acquisition.py @@ -225,10 +225,19 @@ def _scope_fixture_output(self, output: BaseModel, context: ContextPack) -> Base return output async def structured(self, request: ModelRequest[TModel]) -> ModelResult[TModel]: + from tests.fixtures.scripted_hypothesis_responses import ( + hypothesis_output, + table_fill_output, + ) + context = request.context_pack - span_id = context.evidence[0].span_id - if request.role == ModelRole.HYPOTHESIS_GENERATOR: - output: BaseModel = HypothesisPortfolio( + span_id = ( + "" if request.role == ModelRole.HYPOTHESIS_TABLE_FILLER else context.evidence[0].span_id + ) + if request.role == ModelRole.HYPOTHESIS_TABLE_FILLER: + output: BaseModel = table_fill_output(request) + elif request.role == ModelRole.HYPOTHESIS_GENERATOR: + output = HypothesisPortfolio( portfolio_id="portfolio:a02", object_id=context.object_id, hypotheses=( @@ -302,7 +311,11 @@ async def structured(self, request: ModelRequest[TModel]) -> ModelResult[TModel] plan_revision_digest=context.input_head_set_digest, ) output = self._scope_fixture_output(output, context) - typed = request.output_model.model_validate(output.model_dump(mode="python")) + typed = ( + hypothesis_output(request, output.model_dump(mode="python")) + if request.role == ModelRole.HYPOTHESIS_GENERATOR + else request.output_model.model_validate(output.model_dump(mode="python")) + ) input_digest = domain_digest( "A02_MODEL_INPUT", "1.0.0", diff --git a/tests/integration/test_current_hero_normal_entry.py b/tests/integration/test_current_hero_normal_entry.py index d640690..fa2b9ec 100644 --- a/tests/integration/test_current_hero_normal_entry.py +++ b/tests/integration/test_current_hero_normal_entry.py @@ -5,15 +5,19 @@ import pytest from qa.scenarios.hero_6g_current_core import run_current_hero -from tests.integration.test_four_projectpack_portability import GenericProjectPackModel +from tests.integration.test_four_projectpack_portability import RecordingGenericProjectPackModel + +from thoth.domain.enums import ModelRole +from thoth.domain.hypothesis import HypothesisPortfolioV3, HypothesisTableFill @pytest.mark.asyncio async def test_current_hero_enters_through_one_normal_thread_input(tmp_path: Path) -> None: + model = RecordingGenericProjectPackModel() result = await run_current_hero( pack_name="6g-sandbox-hero", workspace=tmp_path / "hero", - model=GenericProjectPackModel(), + model=model, execution_mode="SEALED_REPLAY", ) trace = cast(dict[str, object], result.manifest["normal_entry_trace"]) @@ -34,3 +38,19 @@ async def test_current_hero_enters_through_one_normal_thread_input(tmp_path: Pat assert stages["sandbox_outcome"]["terminal_state"] == "COMPLETED" assert stages["revision_memory_receipt"]["semantic_truth_certified"] is False assert result.manifest["external_write_performed"] is False + generators = [call for call in model.calls if call.role == ModelRole.HYPOTHESIS_GENERATOR] + assert generators and all( + call.prompt_version == "hypothesis_portfolio.v3.3" for call in generators + ) + assert any(call.role == ModelRole.HYPOTHESIS_TABLE_FILLER for call in model.calls) + portfolios = [item for item in model.outputs if isinstance(item, HypothesisPortfolioV3)] + fills = [item for item in model.outputs if isinstance(item, HypothesisTableFill)] + assert portfolios and fills + assert all(len(item.hypotheses) == 3 for item in portfolios) + assert all( + len(test.expected_by_hypothesis) == 1 + for item in portfolios + for hypothesis in item.hypotheses + for test in hypothesis.discriminating_tests + ) + assert all(len(test.rows) == 2 for item in fills for test in item.tests) diff --git a/tests/integration/test_four_projectpack_portability.py b/tests/integration/test_four_projectpack_portability.py index ab0de1b..72f236c 100644 --- a/tests/integration/test_four_projectpack_portability.py +++ b/tests/integration/test_four_projectpack_portability.py @@ -1,10 +1,11 @@ from __future__ import annotations from pathlib import Path -from typing import TypeVar +from typing import TypeVar, cast import pytest from pydantic import BaseModel +from tests.fixtures.scripted_hypothesis_responses import hypothesis_output, table_fill_output from thoth.adapters.projectpacks import load_project_pack from thoth.application.workflows.projectpack_run import ProjectPackRunResult @@ -21,6 +22,7 @@ class GenericProjectPackModel: """Domain-neutral QA model: uses only the provided ContextPack, never pack oracle data.""" control_capability = CONTROLLED_MODEL_CONTROL + complete_tables: bool = False async def structured(self, request: ModelRequest[TModel]) -> ModelResult[TModel]: if request.role in { @@ -78,6 +80,8 @@ async def structured(self, request: ModelRequest[TModel]) -> ModelResult[TModel] "status": "TESTABLE", "generated_from_head_set": context.input_head_set_digest, } + elif request.role == ModelRole.HYPOTHESIS_TABLE_FILLER: + payload = table_fill_output(request).model_dump() elif request.role == ModelRole.ACTION_PLANNER: payload = { "plan_id": f"plan:{context.object_id}", @@ -145,7 +149,11 @@ async def structured(self, request: ModelRequest[TModel]) -> ModelResult[TModel] } else: raise AssertionError(f"unexpected generic model role: {request.role}") - output = request.output_model.model_validate(payload) + output = ( + hypothesis_output(request, payload, complete=self.complete_tables) + if request.role == ModelRole.HYPOTHESIS_GENERATOR + else request.output_model.model_validate(payload) + ) output_digest = domain_digest( "GENERIC_PROJECTPACK_MODEL_OUTPUT", "1.0.0", @@ -233,6 +241,21 @@ def _action( } +class RecordingGenericProjectPackModel(GenericProjectPackModel): + """Observe calls without requiring existing subclasses to initialize new parent state.""" + + def __init__(self, *, complete_tables: bool = False) -> None: + self.complete_tables = complete_tables + self.calls: list[ModelRequest[BaseModel]] = [] + self.outputs: list[BaseModel] = [] + + async def structured(self, request: ModelRequest[TModel]) -> ModelResult[TModel]: + self.calls.append(cast(ModelRequest[BaseModel], request)) + result = await super().structured(request) + self.outputs.append(result.output) + return result + + @pytest.mark.asyncio async def test_four_packs_use_same_generic_model_without_oracle_or_cross_domain_leakage( tmp_path: Path, diff --git a/tests/integration/test_research_request_v2.py b/tests/integration/test_research_request_v2.py index f7b40c0..a236efb 100644 --- a/tests/integration/test_research_request_v2.py +++ b/tests/integration/test_research_request_v2.py @@ -6,6 +6,7 @@ import pytest from pydantic import BaseModel +from tests.fixtures.scripted_hypothesis_responses import hypothesis_output, table_fill_output from tests.integration.scoped_runtime import fixture_scope_policy from tests.integration.storage_coverage_helpers import request, value @@ -212,6 +213,8 @@ async def structured[T: BaseModel](self, request: ModelRequest[T]) -> ModelResul next_checks=("Collect a discriminating observation",), uncertainty_reserve="Cause unknown", ) + elif req.role == ModelRole.HYPOTHESIS_TABLE_FILLER: + output = table_fill_output(req) elif req.role == ModelRole.HYPOTHESIS_REVIEWER: output = HypothesisSemanticReview( decisions=tuple( @@ -283,7 +286,11 @@ async def structured[T: BaseModel](self, request: ModelRequest[T]) -> ModelResul ) else: raise AssertionError(req.role) - parsed = req.output_model.model_validate(output.model_dump()) + parsed = ( + hypothesis_output(req, output.model_dump()) + if req.role == ModelRole.HYPOTHESIS_GENERATOR + else req.output_model.model_validate(output.model_dump()) + ) return ModelResult( output=parsed, model_id="CONTROLLED_RESEARCH", diff --git a/tests/unit/test_hypothesis_grounding_replay.py b/tests/unit/test_hypothesis_grounding_replay.py new file mode 100644 index 0000000..cbc52e5 --- /dev/null +++ b/tests/unit/test_hypothesis_grounding_replay.py @@ -0,0 +1,254 @@ +"""Frozen examples test grounding without changing any product generation or storage path.""" + +from __future__ import annotations + +import copy +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError +from scripts.hypothesis_grounding_replay import ( + FIXTURES, + Example, + Proposal, + ReplayContext, + ReplaySuite, + main, + score_row, +) + +from thoth.domain.hypothesis import ExpectedResult + +SUITE = ReplaySuite.model_validate_json((FIXTURES / "cases.json").read_text(encoding="utf-8")) +BY_ID = {example.case_id: example for example in SUITE.examples} +BASELINE = json.loads((FIXTURES / "baseline.json").read_text(encoding="utf-8")) +ROWS = [row for test in BASELINE["payload"]["tests"] for row in test["rows"]] +BASELINE_SHA = "6fffc576d483e4c64ad5ed1b4cd9ea51add280f4b38ccef985405faeeca9ea3b" +CASES_SHA = "2bc6b8bb2ccf1c33e506790eb3cec7fe4e0209a93bbca8476409975b4857d80e" + + +def test_frozen_data_retains_original_rows_and_exact_negative_categories() -> None: + payload = BASELINE["payload"] + encoded = json.dumps( + payload, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode() + assert hashlib.sha256(encoded).hexdigest() == BASELINE["payload_sha256"] == BASELINE_SHA + assert BASELINE["source_sha256"] == ( + "59d3042a31b1059bfc1f5a239742a2606f606e29f908bf78c6c551ecd5d11868" + ) + assert len(payload["hypotheses"]) == 4 and len(payload["tests"]) == 5 and len(ROWS) == 20 + assert all(row["basis"] == [] for row in ROWS) + assert all(ExpectedResult.model_validate(row).model_dump(mode="json") == row for row in ROWS) + assert payload["table_fill"]["state"] == "SKIPPED_FULL" + assert payload["table_fill"]["open_before"] == payload["table_fill"]["open_after"] == 0 + assert payload["dropped_expected_results"] == [] + assert hashlib.sha256((FIXTURES / "cases.json").read_bytes()).hexdigest() == CASES_SHA + assert {case.case_id for case in SUITE.examples if case.category == "NEGATIVE"} == { + "number-only", + "condition-mismatch", + "unit-mismatch", + "denominator-mismatch", + "stale-pinned-revision", + "foreign-or-private-scope", + "invented-span", + "past-observation-to-future-effect", + "counterevidence-as-support", + "synthetic-as-measured", + } + + +@pytest.mark.parametrize("case", SUITE.examples, ids=lambda case: case.case_id) +def test_safe_output_passes_and_adversarial_output_fails_without_mutation(case: Example) -> None: + before = copy.deepcopy(case.model_dump(mode="json")) + good = score_row(case.context, case.accepted) + bad = score_row(case.context, case.rejected) + assert good.passed and good.reasons == () + assert good.false_verification_promotions == good.unauthorized_links == 0 + assert not bad.passed and bad.reasons + assert case.model_dump(mode="json") == before + assert score_row(case.context, case.accepted) == good + + +@pytest.mark.parametrize("index", range(20)) +def test_legacy_twenty_rows_remain_unassessed_without_fabricating_basis(index: int) -> None: + row = ROWS[index] + before = json.dumps(row, ensure_ascii=False, sort_keys=True) + context = ReplayContext( + kind="LEGACY", + project_id="project:baseline", + cutoff_at=datetime(2026, 10, 23, tzinfo=UTC), + data_origin="SYNTHETIC", + legacy_knowledge_state="PREDICTED", + legacy_expected=row["expected"], + ) + proposal = Proposal( + role="UNASSESSED", + knowledge_state="PREDICTED", + expected=row["expected"], + data_origin="SYNTHETIC", + ) + assert score_row(context, proposal).passed + assert json.dumps(row, ensure_ascii=False, sort_keys=True) == before + inferred = proposal.model_copy(update={"role": "HYPOTHESIS_ONLY", "rationale": "guess"}) + assert not score_row(context, inferred).passed # no retroactive role inference + assert not score_row(context, proposal.model_copy(update={"expected": "changed"})).passed + + +@pytest.mark.parametrize( + "name", + [ + "number-only", + "condition-mismatch", + "unit-mismatch", + "denominator-mismatch", + "stale-pinned-revision", + "counterevidence-as-support", + ], +) +def test_corrected_evidence_can_pass_without_changing_the_case_label(name: str) -> None: + bad = BY_ID[name] + actual = BY_ID["exact-rain-observation"] + context = bad.context.model_copy( + update={ + "evidence": actual.context.evidence, + "pinned_sources": actual.context.pinned_sources, + } + ) + proposal = bad.rejected.model_copy( + update={ + "role": "OBSERVATION_RESTATEMENT", + "basis": actual.accepted.basis, + "observation": actual.accepted.observation, + } + ) + assert score_row(context, proposal).passed + + +def test_a_positive_restatement_cannot_pass_by_omitting_or_changing_the_observation() -> None: + case = BY_ID["exact-rain-observation"] + result = score_row(case.context, case.rejected) + assert result.missing_observation_links == 1 + assert "MISSING_REQUIRED_OBSERVATION_LINK" in result.reasons + assert not score_row( + case.context, case.accepted.model_copy(update={"observation": None}) + ).passed + assert case.context.target is not None + changed = case.context.target.model_copy(update={"numerator": 15}) + assert not score_row( + case.context, case.accepted.model_copy(update={"observation": changed}) + ).passed + + +def test_same_project_private_and_foreign_project_are_independent_guards() -> None: + case = BY_ID["exact-rain-observation"] + private = case.context.model_copy(update={"permitted_span_ids": ()}) + assert score_row(private, case.accepted).unauthorized_links == 1 + foreign = case.context.evidence[0].model_copy(update={"project_id": "project:elsewhere"}) + result = score_row(case.context.model_copy(update={"evidence": (foreign,)}), case.accepted) + assert result.unauthorized_links == 1 and "FOREIGN_PROJECT_LINK" in result.reasons + + +def test_cutoff_and_source_bytes_are_checked_without_reading_current_heads() -> None: + case = BY_ID["exact-rain-observation"] + earlier = case.context.model_copy( + update={"cutoff_at": case.context.cutoff_at.replace(year=2025)} + ) + assert "AFTER_CUTOFF" in score_row(earlier, case.accepted).reasons + altered = case.context.evidence[0].model_copy(update={"exact_text": "different bytes"}) + result = score_row(case.context.model_copy(update={"evidence": (altered,)}), case.accepted) + assert "TEXT_HASH_MISMATCH" in result.reasons + # The v1 source stays valid when it is the pinned historical basis; no latest-head I/O. + assert score_row(case.context, case.accepted).passed + + +def test_legacy_links_are_preserved_without_becoming_verified_support() -> None: + case = BY_ID["exact-rain-observation"] + reference = case.context.evidence[0].model_copy(update={"relation": "REFERENCE_ONLY"}) + context = case.context.model_copy( + update={ + "kind": "LEGACY", + "target": None, + "legacy_basis": case.accepted.basis, + "legacy_knowledge_state": case.accepted.knowledge_state, + "legacy_expected": case.accepted.expected, + "evidence": (reference,), + } + ) + proposal = case.accepted.model_copy(update={"role": "UNASSESSED", "observation": None}) + assert score_row(context, proposal).passed + assert not score_row(context, proposal.model_copy(update={"basis": ()})).passed + assert not score_row(context, proposal.model_copy(update={"verification_claim": True})).passed + + +def test_future_unknown_is_distinct_from_a_known_but_unconfirmed_prediction() -> None: + case = BY_ID["past-observation-to-future-effect"] + assert score_row(case.context, case.accepted).passed + unknown = case.accepted.model_copy( + update={ + "role": "UNASSESSED", + "knowledge_state": "UNKNOWN", + "expected": None, + } + ) + assert score_row(case.context, unknown).passed + with pytest.raises(ValidationError): + Proposal.model_validate({**case.accepted.model_dump(), "expected": None}) + + +def test_valid_evidence_cannot_grant_verification_or_duplicate_its_link() -> None: + case = BY_ID["exact-rain-observation"] + promoted = score_row( + case.context, case.accepted.model_copy(update={"verification_claim": True}) + ) + assert promoted.false_verification_promotions == 1 and not promoted.passed + duplicated = case.accepted.model_copy(update={"basis": case.accepted.basis * 2}) + assert "DUPLICATE_LINK" in score_row(case.context, duplicated).reasons + + +def test_grade_covers_every_case_and_returns_failure_on_a_fabricated_link( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + path = tmp_path / "predictions.json" + values = {case.case_id: case.accepted.model_dump(mode="json") for case in SUITE.examples} + path.write_text(json.dumps(values), encoding="utf-8") + assert main(["grade", "--predictions", str(path)]) == 0 + assert json.loads(capsys.readouterr().out)["passed"] is True + values["invented-span"] = BY_ID["invented-span"].rejected.model_dump(mode="json") + path.write_text(json.dumps(values), encoding="utf-8") + assert main(["grade", "--predictions", str(path)]) == 1 + assert json.loads(capsys.readouterr().out)["passed"] is False + del values["exact-rain-observation"] + path.write_text(json.dumps(values), encoding="utf-8") + with pytest.raises(SystemExit) as error: + main(["grade", "--predictions", str(path)]) + assert error.value.code == 2 + + +@pytest.mark.parametrize("corrupt", ["unknown_state", "missing_role", "duplicate_key"]) +def test_invalid_proposals_fail_as_input_errors(tmp_path: Path, corrupt: str) -> None: + values = {case.case_id: case.accepted.model_dump(mode="json") for case in SUITE.examples} + if corrupt == "unknown_state": + values["exact-rain-observation"]["knowledge_state"] = "UNKNOWN" + elif corrupt == "missing_role": + del values["exact-rain-observation"]["role"] + raw = json.dumps(values) + if corrupt == "duplicate_key": + raw = raw[:-1] + ', "exact-rain-observation": {}}' + path = tmp_path / "bad.json" + path.write_text(raw, encoding="utf-8") + with pytest.raises(SystemExit) as error: + main(["grade", "--predictions", str(path)]) + assert error.value.code == 2 + + +def test_an_empty_case_suite_cannot_pass(tmp_path: Path) -> None: + path = tmp_path / "empty.json" + path.write_text( + '{"schema_version":"grounding_replay_cases.v1","examples":[]}', encoding="utf-8" + ) + with pytest.raises(SystemExit) as error: + main(["check-fixtures", "--cases", str(path)]) + assert error.value.code == 2 diff --git a/tests/unit/test_scripted_hypothesis_responses.py b/tests/unit/test_scripted_hypothesis_responses.py new file mode 100644 index 0000000..8aa6299 --- /dev/null +++ b/tests/unit/test_scripted_hypothesis_responses.py @@ -0,0 +1,175 @@ +"""Shared QA models honor requested contracts, including the real open-cell filler gate.""" + +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from typing import Literal + +import pytest +from pydantic import BaseModel +from tests.fixtures.scripted_hypothesis_responses import hypothesis_output +from tests.integration.test_a02_autonomous_acquisition import DynamicA02Model +from tests.integration.test_four_projectpack_portability import ( + GenericProjectPackModel, + RecordingGenericProjectPackModel, +) +from tests.integration.test_research_request_v2 import ControlledResearchModel + +from thoth.application.services.hypothesis_table_filler import fill_tables +from thoth.domain.enums import ModelRole +from thoth.domain.evidence import EvidenceSpan +from thoth.domain.hypothesis import HypothesisPortfolio, HypothesisPortfolioV3, HypothesisTableFill +from thoth.domain.hypothesis_v4 import HypothesisPortfolioV4, HypothesisTableFillV4 +from thoth.domain.model import ContextPack, ModelRequest + +CUTOFF = datetime(2026, 10, 23, tzinfo=UTC) + + +def model_for(name: str): + if name == "generic": + return GenericProjectPackModel() + if name == "dynamic": + return DynamicA02Model() + return ControlledResearchModel(one=True, tested=True) + + +def request_for[T: BaseModel]( + output: type[T], role: ModelRole, language: Literal["ko", "en"] +) -> ModelRequest[T]: + span = EvidenceSpan.model_validate( + { + "span_id": "span:scripted", + "project_id": "project:scripted", + "artifact_id": "artifact:scripted", + "source_version_id": "source:scripted", + "locator": {"line": 1}, + "exact_text": "synthetic", + "text_sha256": "a" * 64, + "extraction_method": "synthetic", + "support_state": "EXTRACTED", + "authority_state": "UNCLASSIFIED", + "verification_state": "NOT_CHECKED", + "cutoff_state": "ELIGIBLE", + } + ) + context = ContextPack( + case_id="case:scripted", + project_id="project:scripted", + object_id="object:scripted", + problem="원인을 비교해 주세요", + evidence=(span,), + criteria=(), + sufficiency=None, + input_head_set_digest="b" * 64, + ) + return ModelRequest( + role=role, + project_id=context.project_id, + cutoff_at=CUTOFF, + context_pack=context, + output_model=output, + prompt_version="controlled-contract", + model_policy_ref="model-policy:scripted", + max_output_tokens=1000, + response_language=language, + ) + + +@pytest.mark.parametrize("name", ["generic", "dynamic", "controlled"]) +@pytest.mark.parametrize( + "output", [HypothesisPortfolio, HypothesisPortfolioV3, HypothesisPortfolioV4] +) +@pytest.mark.parametrize("language", ["ko", "en"]) +async def test_models_generate_the_requested_version_with_stable_ids( + name: str, output: type[HypothesisPortfolio], language: Literal["ko", "en"] +) -> None: + model = model_for(name) + request = request_for(output, ModelRole.HYPOTHESIS_GENERATOR, language) + result = await model.structured(request) + assert type(result.output) is output + assert len(result.output.hypotheses) == (1 if name == "controlled" else 3) + for owner in result.output.hypotheses: + if isinstance(result.output, HypothesisPortfolioV3): + row = owner.model_dump(mode="json")["discriminating_tests"][0][ + "expected_by_hypothesis" + ][0] + assert row["hypothesis_id"] == owner.hypothesis_id and row["basis"] == [] + assert ("knowledge_state" in row) is (output is HypothesisPortfolioV4) + assert ("차이" in row["expected"]) is (language == "ko") + else: + assert "refutation_conditions" not in owner.model_dump() + + +@pytest.mark.parametrize("name", ["generic", "dynamic", "controlled"]) +@pytest.mark.parametrize("output", [HypothesisTableFill, HypothesisTableFillV4]) +async def test_models_answer_only_the_requested_open_cells( + name: str, output: type[HypothesisTableFill] +) -> None: + request = request_for(output, ModelRole.HYPOTHESIS_TABLE_FILLER, "en") + context = request.context_pack.model_copy( + update={ + "evidence": (), + "research_context": { + "table_filler": { + "tests": [ + { + "designed_for": "h1", + "test_id": "t1", + "expected_if_true": "different", + "expected_if_alternative": "same", + "open_hypothesis_ids": ["h2", "h3"], + } + ] + } + }, + } + ) + result = await model_for(name).structured(replace(request, context_pack=context)) + assert type(result.output) is output + (test,) = result.output.tests + assert test.designed_for == "h1" and test.test_id == "t1" + assert [(row.hypothesis_id, row.expected, row.basis) for row in test.rows] == [ + ("h2", "same", ()), + ("h3", "same", ()), + ] + + +@dataclass(frozen=True) +class FillCommand: + cutoff_at: datetime + model_policy_ref: str + evidence: tuple[EvidenceSpan, ...] + + +@pytest.mark.parametrize("complete", [False, True]) +async def test_product_calls_filler_only_for_open_tables(complete: bool) -> None: + model = RecordingGenericProjectPackModel(complete_tables=complete) + request = request_for(HypothesisPortfolioV3, ModelRole.HYPOTHESIS_GENERATOR, "en") + generated = await model.structured(request) + context = request.context_pack.model_copy( + update={"research_context": {"response_language": "en"}} + ) + filled = await fill_tables( + model, + generated.output, + context, + FillCommand(CUTOFF, request.model_policy_ref, context.evidence), + ) + assert isinstance(filled, HypothesisPortfolioV3) and filled.table_fill is not None + assert filled.table_fill.state == ("SKIPPED_FULL" if complete else "CALLED") + assert sum(call.role == ModelRole.HYPOTHESIS_TABLE_FILLER for call in model.calls) == ( + 0 if complete else 1 + ) + assert ( + sum( + len(test.expected_by_hypothesis) + for h in filled.hypotheses + for test in h.discriminating_tests + ) + == 9 + ) + + +async def test_v2_payload_is_not_enriched() -> None: + request = request_for(HypothesisPortfolio, ModelRole.HYPOTHESIS_GENERATOR, "en") + original = (await GenericProjectPackModel().structured(request)).output + assert hypothesis_output(request, original.model_dump()).model_dump() == original.model_dump()