[AIGTWY-5288] Stop requiring fixed text in inference smoke tests

tests/README.md

@@ -654,7 +654,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi
 
 | Test | User action | Expected evidence |
 | --- | --- | --- |
-| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies both native MPS APIs, generated settings, and sandbox inclusion with web_search excluded; each TUI case records its exact MPS target header and model, paired HTTP 200 response, and final marker. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed |
+| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies both native MPS APIs, generated settings, and sandbox inclusion with web_search excluded; each TUI case records its exact MPS target header and model, paired HTTP 200 response, and a completed parent turn with a nonempty greeting answer. Request IDs correlate traffic without requiring the model to repeat them. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed |
 | `test_ug_configure_claude_databricks` | Configure Databricks Hosted; execute the generated auth helper; launch plain `ug claude`, read a file, and open `/model` | Generated helper invokes `ug` with clean token stdout; assistant returns an unpredictable file value; native discovery caches `system.ai` models and the picker shows a discovered model without an opt-in flag; normal exit; reopen with working keyboard input |
 | `test_ug_configure_claude_anthropic_mps` | Select Anthropic MPS in the real configure picker; launch Claude | Saved provider in status; completed TUI file task; normal exit |
 | `test_ug_configure_claude_bedrock_mps` | Select the restricted Bedrock MPS in the real configure picker; launch Claude without an explicit model | Saved provider in status; Sonnet 4.6 is selected from the Sonnet + Haiku allow-list and completes a TUI file task; normal exit |
@@ -751,7 +751,13 @@ one configuration case and separate Codex and Claude TUI task cases. The workspa
 preconfigured for CUJ2 and is read-only; every case validates it, and the fixture compares
 it again during teardown. The TUI cases forward agent requests through the shared per-test
 loopback recorder to the real workspace and assert the provider-service header, model,
-paired HTTP 200 response, and final answer. The configuration case verifies sandbox MCP
+paired HTTP 200 response, and a nonempty final answer in the completed parent turn matching
+the submitted greeting prompt. Request IDs correlate recorded traffic without constraining
+answer text. Offline evidence checks reject empty, incomplete, unrelated and child-session
+answers while retaining hidden-value requirements for file tasks.
+The existing direct Anthropic non-relay MPS e2e check also requires a complete Messages
+response: assistant role/type, the pinned model, `end_turn`, and nonempty text.
+The configuration case verifies sandbox MCP
 selection and generated client listings; live sandbox execution remains deferred because
 MAS cannot downscope the CI service principal. CUJ1 and CUJ2 also initialize Claude Desktop's
 Linux profile path and verify that its profile uses, respectively, the published Databricks-hosted

tests/e2e_cuj/README.md

@@ -81,6 +81,11 @@ aliases use names such as `ug_e2e.models.claude_haiku`, which Desktop's Claude-f
 not accept, so existing files must remain unchanged and UG must not save selection state.
 Successful Desktop profiles remain covered by CUJ1/CUJ2. No extra CUJ test or configure is added.
 
+CUJ2's Claude and Codex MPS inference cases ask for a short greeting. Each requires a completed
+native parent turn matching that prompt, a nonempty answer, the expected provider/model and
+its paired HTTP 200 response. Random request IDs correlate traffic without requiring models
+to repeat them. File, skill, MCP and subagent tasks keep their expected-value assertions.
+
 ## Smart-routing CUJ
 
 `test_cuj4_smart_routing.py` uses its own read-only workspace with managed smart routing enabled.

tests/e2e_cuj/helpers/evidence.py

@@ -117,6 +117,11 @@ def message_text(content):
     )
 
 
+def _answer_matches(task, answer):
+    """task.value=None accepts any nonempty final answer."""
+    return bool(answer.strip()) and (task.value is None or task.value in answer)
+
+
 @dataclass
 class CompletedTurn:
     session_id: str
@@ -190,7 +195,7 @@ def _completed_turn(records, task):
         last = responses[-1]
         answer = message_text(last.get("content"))
         if (
-            task.value not in answer
+            not _answer_matches(task, answer)
             or not last.get("id")
             or any(part.get("type") == "tool_use" for part in last.get("content", []))
             or last.get("stop_reason") not in (None, "end_turn", "stop_sequence")
@@ -275,7 +280,7 @@ def _completed_turn(records, task):
         if not prompt_sources or not target_turn or started_turn != target_turn or answer is None:
             return None
         models = contexts.get(target_turn)
-        if task.value not in answer or not models:
+        if not _answer_matches(task, answer) or not models:
             return None
         assert all(models), "Missing native turn model metadata"
         return CompletedTurn(meta[0]["id"], target_turn, models, answer)
@@ -291,6 +296,7 @@ def get_cuj_helper(agent):
 
 
 def completed_turn(agent, records, task):
+    """Match a completed turn; task.value=None accepts any nonempty final answer."""
     return get_cuj_helper(agent)._completed_turn(records, task)
 
 

tests/e2e_cuj/test_cuj2_mps_mcp.py

@@ -5,15 +5,12 @@
 import json
 import shutil
 import tomllib
+from dataclasses import asdict
+from types import SimpleNamespace
 
 from databricks.sdk.service.catalog import ListModelProviderServicesRequestView
 
-from tests.integration.utils.evidence import (
-    agent_sessions,
-    assistant_answers,
-    fresh_id,
-    is_child_session,
-)
+from tests.integration.utils.evidence import fresh_id
 
 from .base import BaseCujTest
 from .helpers.constants import (
@@ -29,7 +26,7 @@
     WORKSPACE_POLICY_PATH,
 )
 from .helpers.desktop import assert_claude_desktop_profile, enable_claude_desktop
-from .helpers.evidence import assert_served
+from .helpers.evidence import SessionEvidence, assert_served
 from .helpers.mcp import registered_name
 from .helpers.poll import poll
 from .helpers.terminal import Terminal
@@ -172,37 +169,8 @@ def listing(binary=binary):
         assert not _mcp_name_listed(output, WEB_SEARCH_MCP_SERVICE_NAME), output
 
 
-def _parent_transcripts(session, agent: str) -> dict[str, list[dict]]:
-    return {
-        path: records
-        for path, records in agent_sessions(session, agent).items()
-        if not is_child_session(agent, path, records)
-    }
-
-
-def _task_complete(session, agent: str, marker: str) -> bool:
-    transcripts = _parent_transcripts(session, agent)
-    return any(
-        marker in answer
-        for records in transcripts.values()
-        for answer in assistant_answers(agent, records)
-    )
-
-
-def _assert_task_evidence(session, agent: str, marker: str) -> None:
-    transcripts = _parent_transcripts(session, agent)
-    answers = [
-        answer for records in transcripts.values() for answer in assistant_answers(agent, records)
-    ]
-    assert any(marker in answer for answer in answers), answers
-    session.record(
-        f"cuj2-{agent}-evidence",
-        {"marker": marker, "answers": answers},
-    )
-
-
 def _agent_prompt(marker: str) -> str:
-    return f"Reply with the exact marker `{marker}` and no other text. Do not use tools."
+    return f"Reply with a short greeting. Do not use tools. Request ID: {marker}."
 
 
 def _served_marker_request(recorder, checkpoint: int, agent: str, marker: str):
@@ -284,7 +252,7 @@ def test_cuj_codex_inference(self, cuj):
 
         Expected: Codex sends a Responses request for gpt-5-nano with the
         ug_e2e.providers.openai target header and receives its paired HTTP 200 response; its
-        parent transcript contains the final marker.
+        parent transcript contains a completed turn with a nonempty answer to the greeting prompt.
         """
         session, workspace, recorder = cuj
         published = workspace.config()
@@ -305,11 +273,13 @@ def test_cuj_codex_inference(self, cuj):
 
         recorder.prepare_launch()
         marker = f"CUJ2-CODEX-{fresh_id()}"
+        task = SimpleNamespace(prompt=_agent_prompt(marker), value=None)
+        evidence = SessionEvidence(session.home, CODEX)
         checkpoint = recorder.checkpoint()
         with Terminal(session, "cuj2-codex", [CODEX]) as tui:
             tui.boot()
-            tui.submit(_agent_prompt(marker))
-            tui.wait_until(lambda: _task_complete(session, CODEX, marker), "completed Codex task")
+            tui.submit(task.prompt)
+            tui.wait_until(lambda: evidence.completed(task), "completed Codex task")
             tui.exit_normally()
 
         request = recorder.expect_request(
@@ -325,7 +295,9 @@ def test_cuj_codex_inference(self, cuj):
             model=MODEL_PROVIDER_SERVICE_FIXTURES[CODEX][1],
             marker=marker,
         )
-        _assert_task_evidence(session, CODEX, marker)
+        turn = evidence.completed(task)
+        assert turn is not None, "No completed Codex answer matched the greeting prompt"
+        session.record("cuj2-codex-evidence", {"marker": marker, **asdict(turn)})
 
 
 class TestCuj2ClaudeInference(_Cuj2Base):
@@ -334,7 +306,7 @@ def test_cuj_claude_inference(self, cuj):
 
         Expected: Claude sends a Messages request for claude-haiku-4-5-20251001 with the
         ug_e2e.providers.anthropic target header and receives its paired HTTP 200 response; its
-        parent transcript contains the final marker.
+        parent transcript contains a completed turn with a nonempty answer to the greeting prompt.
         """
         session, workspace, recorder = cuj
         published = workspace.config()
@@ -355,11 +327,13 @@ def test_cuj_claude_inference(self, cuj):
 
         recorder.prepare_launch()
         marker = f"CUJ2-CLAUDE-{fresh_id()}"
+        task = SimpleNamespace(prompt=_agent_prompt(marker), value=None)
+        evidence = SessionEvidence(session.home, CLAUDE)
         checkpoint = recorder.checkpoint()
         with Terminal(session, "cuj2-claude", [CLAUDE]) as tui:
             tui.boot()
-            tui.submit(_agent_prompt(marker))
-            tui.wait_until(lambda: _task_complete(session, CLAUDE, marker), "completed Claude task")
+            tui.submit(task.prompt)
+            tui.wait_until(lambda: evidence.completed(task), "completed Claude task")
             tui.exit_normally()
 
         recorder.expect_request(
@@ -376,4 +350,6 @@ def test_cuj_claude_inference(self, cuj):
             model=MODEL_PROVIDER_SERVICE_FIXTURES[CLAUDE][1],
             marker=marker,
         )
-        _assert_task_evidence(session, CLAUDE, marker)
+        turn = evidence.completed(task)
+        assert turn is not None, "No completed Claude answer matched the greeting prompt"
+        session.record("cuj2-claude-evidence", {"marker": marker, **asdict(turn)})

tests/integration/README.md

@@ -91,7 +91,11 @@ Its Claude/Codex evidence helpers keep scenario-specific assertions separate fro
 Workspace config/catalog reads use its base class's Databricks SDK client. Configuration
 is read-only and checked for changes at teardown; concurrent readers need no reservation.
 CUJ2 adds three separately collected cases for exact published MPS/MCP configuration, Codex
-inference, and Claude inference. The config equality also accounts for the workspace's fixture
+inference, and Claude inference. Both inference cases ask for a short greeting and require a
+completed parent turn matching the submitted prompt with a nonempty answer, plus the expected
+provider/model and paired HTTP 200. Random IDs correlate requests; answers need not repeat them.
+File, skill, MCP and subagent tasks retain their expected-value checks.
+The config equality also accounts for the workspace's fixture
 skill name as data; skill download and invocation are not covered.
 
 The dedicated smart-routing CUJ in `../e2e_cuj/test_cuj4_smart_routing.py` runs the original

tests/test_cuj_evidence.py

@@ -146,6 +146,26 @@ def test_cuj_evidence_completed_native_turn(tmp_path, agent):
     assert result["selected_model"] == model
 
 
+@pytest.mark.parametrize("agent", [CLAUDE, CODEX])
+@pytest.mark.parametrize("answer", ["Hello!", "", " \n "])
+def test_cuj_inference_requires_nonempty_final_answer(agent, answer):
+    task = SimpleNamespace(prompt="Reply with a short greeting. Request ID: unique-id.", value=None)
+    rows = records(agent, SimpleNamespace(prompt=task.prompt, value=answer), "model")
+    turn = completed_turn(agent, rows, task)
+    if answer.strip():
+        assert turn is not None
+        assert turn.answer == answer
+    else:
+        assert turn is None
+
+
+@pytest.mark.parametrize("agent", [CLAUDE, CODEX])
+def test_cuj_file_task_still_requires_hidden_value(tmp_path, agent):
+    task = FileTask(SimpleNamespace(cwd=tmp_path))
+    rows = records(agent, SimpleNamespace(prompt=task.prompt, value="Hello!"), "model")
+    assert completed_turn(agent, rows, task) is None
+
+
 def test_cuj_evidence_claude_real_user_message_ends_parent_turn():
     task = SimpleNamespace(
         prompt="Delegate reading the file to one subagent.", value="hidden-value"
@@ -165,6 +185,7 @@ def test_cuj_evidence_claude_real_user_message_ends_parent_turn():
 
 
 @pytest.mark.parametrize("agent", [CLAUDE, CODEX])
+@pytest.mark.parametrize("require_value", [True, False], ids=["file", "inference"])
 @pytest.mark.parametrize(
     "failure",
     [
@@ -174,11 +195,13 @@ def test_cuj_evidence_claude_real_user_message_ends_parent_turn():
         "child",
     ],
 )
-def test_cuj_evidence_rejects_false_positives(tmp_path, agent, failure):
+def test_cuj_evidence_rejects_false_positives(tmp_path, agent, require_value, failure):
     task = FileTask(SimpleNamespace(cwd=tmp_path))
     model = {CLAUDE: "system.ai.claude-opus-4-8", CODEX: "system.ai.gpt-6-sol"}[agent]
     boundary = SessionEvidence(tmp_path, agent)
     rows = records(agent, task, model)
+    if not require_value:
+        task.value = None
     if failure == "no_prompt":
         rows = [
             row
@@ -204,9 +227,12 @@ def test_cuj_evidence_rejects_false_positives(tmp_path, agent, failure):
         )
 
 
-def test_cuj_evidence_codex_rejects_wrong_turn(tmp_path):
+@pytest.mark.parametrize("require_value", [True, False], ids=["file", "inference"])
+def test_cuj_evidence_codex_rejects_wrong_turn(tmp_path, require_value):
     task = FileTask(SimpleNamespace(cwd=tmp_path))
     rows = records(CODEX, task, "gpt-6-sol")
+    if not require_value:
+        task.value = None
     wrong_turn = copy.deepcopy(rows)
     wrong_turn[-1]["payload"]["turn_id"] = "different-turn"
     assert completed_turn(CODEX, wrong_turn, task) is None

tests/test_e2e.py

@@ -796,7 +796,7 @@ class TestAnthropicNonRelayMps:
     def _messages_request(workspace: str, token: str, model: str) -> httpx.Response:
         body = {
             "model": model,
-            "max_tokens": 16,
+            "max_tokens": 64,  # Leave room to finish without truncating the greeting.
             "stream": False,
             "messages": [{"role": "user", "content": "say hi in 5 words or less"}],
         }
@@ -825,7 +825,13 @@ def test_direct_inference_routes_to_haiku(self, e2e_workspace, e2e_token):
         resp = self._messages_request(e2e_workspace, e2e_token, CI_ANTHROPIC_MODEL)
         self._skip_if_unusable(resp)
         assert resp.status_code == 200, f"HTTP {resp.status_code}: {resp.text[:300]}"
-        assert resp.json().get("model") == CI_ANTHROPIC_MODEL
+        message = resp.json()
+        assert message["type"] == "message" and message["role"] == "assistant"
+        assert message["model"] == CI_ANTHROPIC_MODEL
+        assert message["stop_reason"] == "end_turn"
+        assert "".join(
+            part["text"] for part in message["content"] if part["type"] == "text"
+        ).strip()
 
     def test_rejects_model_outside_allowlist(self, e2e_workspace, e2e_token):
         # The MPS enforces its allowlist up front: a model it doesn't declare is refused.