From 71692487b37c1c6896b45839e63824ad8c20de8b Mon Sep 17 00:00:00 2001 From: Mason Cao Date: Thu, 29 Oct 2026 23:45:03 +0000 Subject: [PATCH 1/2] [AIGTWY-5288] Validate inference without fixed response text --- tests/README.md | 10 ++++- tests/e2e_cuj/README.md | 5 +++ tests/e2e_cuj/helpers/evidence.py | 10 ++++- tests/e2e_cuj/test_cuj2_mps_mcp.py | 66 ++++++++++-------------------- tests/integration/README.md | 6 ++- tests/test_cuj_evidence.py | 30 +++++++++++++- tests/test_e2e.py | 8 +++- 7 files changed, 82 insertions(+), 53 deletions(-) diff --git a/tests/README.md b/tests/README.md index a78179707..5e5698fe1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -654,7 +654,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | Test | User action | Expected evidence | | --- | --- | --- | -| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies both native MPS APIs, generated settings, and sandbox inclusion with web_search excluded; each TUI case records its exact MPS target header and model, paired HTTP 200 response, and final marker. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed | +| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies both native MPS APIs, generated settings, and sandbox inclusion with web_search excluded; each TUI case records its exact MPS target header and model, paired HTTP 200 response, and a completed parent turn with a nonempty greeting answer. Request IDs correlate traffic without requiring the model to repeat them. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed | | `test_ug_configure_claude_databricks` | Configure Databricks Hosted; execute the generated auth helper; launch plain `ug claude`, read a file, and open `/model` | Generated helper invokes `ug` with clean token stdout; assistant returns an unpredictable file value; native discovery caches `system.ai` models and the picker shows a discovered model without an opt-in flag; normal exit; reopen with working keyboard input | | `test_ug_configure_claude_anthropic_mps` | Select Anthropic MPS in the real configure picker; launch Claude | Saved provider in status; completed TUI file task; normal exit | | `test_ug_configure_claude_bedrock_mps` | Select the restricted Bedrock MPS in the real configure picker; launch Claude without an explicit model | Saved provider in status; Sonnet 4.6 is selected from the Sonnet + Haiku allow-list and completes a TUI file task; normal exit | @@ -751,7 +751,13 @@ one configuration case and separate Codex and Claude TUI task cases. The workspa preconfigured for CUJ2 and is read-only; every case validates it, and the fixture compares it again during teardown. The TUI cases forward agent requests through the shared per-test loopback recorder to the real workspace and assert the provider-service header, model, -paired HTTP 200 response, and final answer. The configuration case verifies sandbox MCP +paired HTTP 200 response, and a nonempty final answer in the completed parent turn matching +the submitted greeting prompt. Request IDs correlate recorded traffic without constraining +answer text. Offline evidence checks reject empty, incomplete, unrelated and child-session +answers while retaining hidden-value requirements for file tasks. +The existing direct Anthropic non-relay MPS e2e check also requires a complete Messages +response: assistant role/type, the pinned model, `end_turn`, and nonempty text. +The configuration case verifies sandbox MCP selection and generated client listings; live sandbox execution remains deferred because MAS cannot downscope the CI service principal. CUJ1 and CUJ2 also initialize Claude Desktop's Linux profile path and verify that its profile uses, respectively, the published Databricks-hosted diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 0d4e196bd..3a1b8448c 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -81,6 +81,11 @@ aliases use names such as `ug_e2e.models.claude_haiku`, which Desktop's Claude-f not accept, so existing files must remain unchanged and UG must not save selection state. Successful Desktop profiles remain covered by CUJ1/CUJ2. No extra CUJ test or configure is added. +CUJ2's Claude and Codex MPS inference cases ask for a short greeting. Each requires a completed +native parent turn matching that prompt, a nonempty answer, the expected provider/model and +its paired HTTP 200 response. Random request IDs correlate traffic without requiring models +to repeat them. File, skill, MCP and subagent tasks keep their expected-value assertions. + ## Smart-routing CUJ `test_cuj4_smart_routing.py` uses its own read-only workspace with managed smart routing enabled. diff --git a/tests/e2e_cuj/helpers/evidence.py b/tests/e2e_cuj/helpers/evidence.py index 5140563a6..11e8f3911 100644 --- a/tests/e2e_cuj/helpers/evidence.py +++ b/tests/e2e_cuj/helpers/evidence.py @@ -190,7 +190,8 @@ def _completed_turn(records, task): last = responses[-1] answer = message_text(last.get("content")) if ( - task.value not in answer + not answer.strip() + or (task.value is not None and task.value not in answer) or not last.get("id") or any(part.get("type") == "tool_use" for part in last.get("content", [])) or last.get("stop_reason") not in (None, "end_turn", "stop_sequence") @@ -275,7 +276,11 @@ def _completed_turn(records, task): if not prompt_sources or not target_turn or started_turn != target_turn or answer is None: return None models = contexts.get(target_turn) - if task.value not in answer or not models: + if ( + not answer.strip() + or (task.value is not None and task.value not in answer) + or not models + ): return None assert all(models), "Missing native turn model metadata" return CompletedTurn(meta[0]["id"], target_turn, models, answer) @@ -291,6 +296,7 @@ def get_cuj_helper(agent): def completed_turn(agent, records, task): + """Match a completed turn; task.value=None accepts any nonempty final answer.""" return get_cuj_helper(agent)._completed_turn(records, task) diff --git a/tests/e2e_cuj/test_cuj2_mps_mcp.py b/tests/e2e_cuj/test_cuj2_mps_mcp.py index c88331f98..8c30fdea4 100644 --- a/tests/e2e_cuj/test_cuj2_mps_mcp.py +++ b/tests/e2e_cuj/test_cuj2_mps_mcp.py @@ -5,15 +5,12 @@ import json import shutil import tomllib +from dataclasses import asdict +from types import SimpleNamespace from databricks.sdk.service.catalog import ListModelProviderServicesRequestView -from tests.integration.utils.evidence import ( - agent_sessions, - assistant_answers, - fresh_id, - is_child_session, -) +from tests.integration.utils.evidence import fresh_id from .base import BaseCujTest from .helpers.constants import ( @@ -29,7 +26,7 @@ WORKSPACE_POLICY_PATH, ) from .helpers.desktop import assert_claude_desktop_profile, enable_claude_desktop -from .helpers.evidence import assert_served +from .helpers.evidence import SessionEvidence, assert_served from .helpers.mcp import registered_name from .helpers.poll import poll from .helpers.terminal import Terminal @@ -172,37 +169,8 @@ def listing(binary=binary): assert not _mcp_name_listed(output, WEB_SEARCH_MCP_SERVICE_NAME), output -def _parent_transcripts(session, agent: str) -> dict[str, list[dict]]: - return { - path: records - for path, records in agent_sessions(session, agent).items() - if not is_child_session(agent, path, records) - } - - -def _task_complete(session, agent: str, marker: str) -> bool: - transcripts = _parent_transcripts(session, agent) - return any( - marker in answer - for records in transcripts.values() - for answer in assistant_answers(agent, records) - ) - - -def _assert_task_evidence(session, agent: str, marker: str) -> None: - transcripts = _parent_transcripts(session, agent) - answers = [ - answer for records in transcripts.values() for answer in assistant_answers(agent, records) - ] - assert any(marker in answer for answer in answers), answers - session.record( - f"cuj2-{agent}-evidence", - {"marker": marker, "answers": answers}, - ) - - def _agent_prompt(marker: str) -> str: - return f"Reply with the exact marker `{marker}` and no other text. Do not use tools." + return f"Reply with a short greeting. Do not use tools. Request ID: {marker}." def _served_marker_request(recorder, checkpoint: int, agent: str, marker: str): @@ -284,7 +252,7 @@ def test_cuj_codex_inference(self, cuj): Expected: Codex sends a Responses request for gpt-5-nano with the ug_e2e.providers.openai target header and receives its paired HTTP 200 response; its - parent transcript contains the final marker. + parent transcript contains a completed turn with a nonempty answer to the greeting prompt. """ session, workspace, recorder = cuj published = workspace.config() @@ -305,11 +273,13 @@ def test_cuj_codex_inference(self, cuj): recorder.prepare_launch() marker = f"CUJ2-CODEX-{fresh_id()}" + task = SimpleNamespace(prompt=_agent_prompt(marker), value=None) + evidence = SessionEvidence(session.home, CODEX) checkpoint = recorder.checkpoint() with Terminal(session, "cuj2-codex", [CODEX]) as tui: tui.boot() - tui.submit(_agent_prompt(marker)) - tui.wait_until(lambda: _task_complete(session, CODEX, marker), "completed Codex task") + tui.submit(task.prompt) + tui.wait_until(lambda: evidence.completed(task), "completed Codex task") tui.exit_normally() request = recorder.expect_request( @@ -325,7 +295,9 @@ def test_cuj_codex_inference(self, cuj): model=MODEL_PROVIDER_SERVICE_FIXTURES[CODEX][1], marker=marker, ) - _assert_task_evidence(session, CODEX, marker) + turn = evidence.completed(task) + assert turn is not None, "No completed Codex answer matched the greeting prompt" + session.record("cuj2-codex-evidence", {"marker": marker, **asdict(turn)}) class TestCuj2ClaudeInference(_Cuj2Base): @@ -334,7 +306,7 @@ def test_cuj_claude_inference(self, cuj): Expected: Claude sends a Messages request for claude-haiku-4-5-20251001 with the ug_e2e.providers.anthropic target header and receives its paired HTTP 200 response; its - parent transcript contains the final marker. + parent transcript contains a completed turn with a nonempty answer to the greeting prompt. """ session, workspace, recorder = cuj published = workspace.config() @@ -355,11 +327,13 @@ def test_cuj_claude_inference(self, cuj): recorder.prepare_launch() marker = f"CUJ2-CLAUDE-{fresh_id()}" + task = SimpleNamespace(prompt=_agent_prompt(marker), value=None) + evidence = SessionEvidence(session.home, CLAUDE) checkpoint = recorder.checkpoint() with Terminal(session, "cuj2-claude", [CLAUDE]) as tui: tui.boot() - tui.submit(_agent_prompt(marker)) - tui.wait_until(lambda: _task_complete(session, CLAUDE, marker), "completed Claude task") + tui.submit(task.prompt) + tui.wait_until(lambda: evidence.completed(task), "completed Claude task") tui.exit_normally() recorder.expect_request( @@ -376,4 +350,6 @@ def test_cuj_claude_inference(self, cuj): model=MODEL_PROVIDER_SERVICE_FIXTURES[CLAUDE][1], marker=marker, ) - _assert_task_evidence(session, CLAUDE, marker) + turn = evidence.completed(task) + assert turn is not None, "No completed Claude answer matched the greeting prompt" + session.record("cuj2-claude-evidence", {"marker": marker, **asdict(turn)}) diff --git a/tests/integration/README.md b/tests/integration/README.md index e249c9e07..aeb1255ad 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -91,7 +91,11 @@ Its Claude/Codex evidence helpers keep scenario-specific assertions separate fro Workspace config/catalog reads use its base class's Databricks SDK client. Configuration is read-only and checked for changes at teardown; concurrent readers need no reservation. CUJ2 adds three separately collected cases for exact published MPS/MCP configuration, Codex -inference, and Claude inference. The config equality also accounts for the workspace's fixture +inference, and Claude inference. Both inference cases ask for a short greeting and require a +completed parent turn matching the submitted prompt with a nonempty answer, plus the expected +provider/model and paired HTTP 200. Random IDs correlate requests; answers need not repeat them. +File, skill, MCP and subagent tasks retain their expected-value checks. +The config equality also accounts for the workspace's fixture skill name as data; skill download and invocation are not covered. The dedicated smart-routing CUJ in `../e2e_cuj/test_cuj4_smart_routing.py` runs the original diff --git a/tests/test_cuj_evidence.py b/tests/test_cuj_evidence.py index 65824c74c..b3dc82ff5 100644 --- a/tests/test_cuj_evidence.py +++ b/tests/test_cuj_evidence.py @@ -146,6 +146,26 @@ def test_cuj_evidence_completed_native_turn(tmp_path, agent): assert result["selected_model"] == model +@pytest.mark.parametrize("agent", [CLAUDE, CODEX]) +@pytest.mark.parametrize("answer", ["Hello!", "Hi there.", "", " \n "]) +def test_cuj_inference_requires_nonempty_final_answer(agent, answer): + task = SimpleNamespace(prompt="Reply with a short greeting. Request ID: unique-id.", value=None) + rows = records(agent, SimpleNamespace(prompt=task.prompt, value=answer), "model") + turn = completed_turn(agent, rows, task) + if answer.strip(): + assert turn is not None + assert turn.answer == answer + else: + assert turn is None + + +@pytest.mark.parametrize("agent", [CLAUDE, CODEX]) +def test_cuj_file_task_still_requires_hidden_value(tmp_path, agent): + task = FileTask(SimpleNamespace(cwd=tmp_path)) + rows = records(agent, SimpleNamespace(prompt=task.prompt, value="Hello!"), "model") + assert completed_turn(agent, rows, task) is None + + def test_cuj_evidence_claude_real_user_message_ends_parent_turn(): task = SimpleNamespace( prompt="Delegate reading the file to one subagent.", value="hidden-value" @@ -165,6 +185,7 @@ def test_cuj_evidence_claude_real_user_message_ends_parent_turn(): @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) +@pytest.mark.parametrize("require_value", [True, False], ids=["file", "inference"]) @pytest.mark.parametrize( "failure", [ @@ -174,11 +195,13 @@ def test_cuj_evidence_claude_real_user_message_ends_parent_turn(): "child", ], ) -def test_cuj_evidence_rejects_false_positives(tmp_path, agent, failure): +def test_cuj_evidence_rejects_false_positives(tmp_path, agent, require_value, failure): task = FileTask(SimpleNamespace(cwd=tmp_path)) model = {CLAUDE: "system.ai.claude-opus-4-8", CODEX: "system.ai.gpt-6-sol"}[agent] boundary = SessionEvidence(tmp_path, agent) rows = records(agent, task, model) + if not require_value: + task.value = None if failure == "no_prompt": rows = [ row @@ -204,9 +227,12 @@ def test_cuj_evidence_rejects_false_positives(tmp_path, agent, failure): ) -def test_cuj_evidence_codex_rejects_wrong_turn(tmp_path): +@pytest.mark.parametrize("require_value", [True, False], ids=["file", "inference"]) +def test_cuj_evidence_codex_rejects_wrong_turn(tmp_path, require_value): task = FileTask(SimpleNamespace(cwd=tmp_path)) rows = records(CODEX, task, "gpt-6-sol") + if not require_value: + task.value = None wrong_turn = copy.deepcopy(rows) wrong_turn[-1]["payload"]["turn_id"] = "different-turn" assert completed_turn(CODEX, wrong_turn, task) is None diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 87eac3ff3..abf105152 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -825,7 +825,13 @@ def test_direct_inference_routes_to_haiku(self, e2e_workspace, e2e_token): resp = self._messages_request(e2e_workspace, e2e_token, CI_ANTHROPIC_MODEL) self._skip_if_unusable(resp) assert resp.status_code == 200, f"HTTP {resp.status_code}: {resp.text[:300]}" - assert resp.json().get("model") == CI_ANTHROPIC_MODEL + message = resp.json() + assert message["type"] == "message" and message["role"] == "assistant" + assert message["model"] == CI_ANTHROPIC_MODEL + assert message["stop_reason"] == "end_turn" + assert "".join( + part["text"] for part in message["content"] if part["type"] == "text" + ).strip() def test_rejects_model_outside_allowlist(self, e2e_workspace, e2e_token): # The MPS enforces its allowlist up front: a model it doesn't declare is refused. From 1bd18528613269a7b7bb25c8225acc090cd7510b Mon Sep 17 00:00:00 2001 From: Mason Cao Date: Fri, 30 Oct 2026 00:48:56 +0000 Subject: [PATCH 2/2] [AIGTWY-5288] Address inference test review feedback --- tests/e2e_cuj/helpers/evidence.py | 14 +++++++------- tests/test_cuj_evidence.py | 2 +- tests/test_e2e.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/e2e_cuj/helpers/evidence.py b/tests/e2e_cuj/helpers/evidence.py index 11e8f3911..bb31a96f8 100644 --- a/tests/e2e_cuj/helpers/evidence.py +++ b/tests/e2e_cuj/helpers/evidence.py @@ -117,6 +117,11 @@ def message_text(content): ) +def _answer_matches(task, answer): + """task.value=None accepts any nonempty final answer.""" + return bool(answer.strip()) and (task.value is None or task.value in answer) + + @dataclass class CompletedTurn: session_id: str @@ -190,8 +195,7 @@ def _completed_turn(records, task): last = responses[-1] answer = message_text(last.get("content")) if ( - not answer.strip() - or (task.value is not None and task.value not in answer) + not _answer_matches(task, answer) or not last.get("id") or any(part.get("type") == "tool_use" for part in last.get("content", [])) or last.get("stop_reason") not in (None, "end_turn", "stop_sequence") @@ -276,11 +280,7 @@ def _completed_turn(records, task): if not prompt_sources or not target_turn or started_turn != target_turn or answer is None: return None models = contexts.get(target_turn) - if ( - not answer.strip() - or (task.value is not None and task.value not in answer) - or not models - ): + if not _answer_matches(task, answer) or not models: return None assert all(models), "Missing native turn model metadata" return CompletedTurn(meta[0]["id"], target_turn, models, answer) diff --git a/tests/test_cuj_evidence.py b/tests/test_cuj_evidence.py index b3dc82ff5..514e30999 100644 --- a/tests/test_cuj_evidence.py +++ b/tests/test_cuj_evidence.py @@ -147,7 +147,7 @@ def test_cuj_evidence_completed_native_turn(tmp_path, agent): @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) -@pytest.mark.parametrize("answer", ["Hello!", "Hi there.", "", " \n "]) +@pytest.mark.parametrize("answer", ["Hello!", "", " \n "]) def test_cuj_inference_requires_nonempty_final_answer(agent, answer): task = SimpleNamespace(prompt="Reply with a short greeting. Request ID: unique-id.", value=None) rows = records(agent, SimpleNamespace(prompt=task.prompt, value=answer), "model") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index abf105152..8a17baca4 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -796,7 +796,7 @@ class TestAnthropicNonRelayMps: def _messages_request(workspace: str, token: str, model: str) -> httpx.Response: body = { "model": model, - "max_tokens": 16, + "max_tokens": 64, # Leave room to finish without truncating the greeting. "stream": False, "messages": [{"role": "user", "content": "say hi in 5 words or less"}], }