diff --git a/PILOT.md b/PILOT.md index 0c3259a..5922f77 100644 --- a/PILOT.md +++ b/PILOT.md @@ -1792,3 +1792,9 @@ The runner now exposes --retain-private-events to retain bounded raw CLI events Added a synthetic source-only repair fixture with immutable contracts, history, a legacy golden and a consumer path. Three offline guards verify the seeded focused/full failure, independent acceptance of the tenant-aware repair, and rejection of source or evidence tampering. Diagnostic action IDs and causal hypotheses remain distinct. The current contract resolves the apparent legacy conflict on ordinary inspection. This fixture is a local-resolution/abstention control, not evidence of a genuinely unresolved remote choice, native delivery or advisor benefit. No paid comparison was launched. Oracle SHA-256: `212bd0339054ed8af8017adfca84f3154ae019b11138638317a3865accd99e78`. + +## VCR401 — Completed assistant-message delivery measurement + +The shared event reducer now ignores assistant item.started/item.updated draft lifecycle events when measuring message delivery. A completed assistant message remains eligible; a tool executed before completion still makes subsequent acknowledgment late. Two synthetic lifecycle tests and 63 existing collector tests pass. This prevents an unfinished message from poisoning or prematurely satisfying a delivery gate. + +The missing VCR396 acknowledgment cannot be attributed to this issue without its raw events. Historical outcomes remain unchanged. This correction concerns prospective measurement, not advisor efficacy or native Desktop delivery. diff --git a/TASKS.md b/TASKS.md index eac13bb..8b10566 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1196,3 +1196,9 @@ The runner now exposes --retain-private-events to retain bounded raw CLI events Added a synthetic source-only repair fixture with immutable contracts, history, a legacy golden and a consumer path. Three offline guards verify the seeded focused/full failure, independent acceptance of the tenant-aware repair, and rejection of source or evidence tampering. Diagnostic action IDs and causal hypotheses remain distinct. The current contract resolves the apparent legacy conflict on ordinary inspection. This fixture is a local-resolution/abstention control, not evidence of a genuinely unresolved remote choice, native delivery or advisor benefit. No paid comparison was launched. Oracle SHA-256: `212bd0339054ed8af8017adfca84f3154ae019b11138638317a3865accd99e78`. + +## VCR401 — Completed assistant-message delivery measurement + +The shared event reducer now ignores assistant item.started/item.updated draft lifecycle events when measuring message delivery. A completed assistant message remains eligible; a tool executed before completion still makes subsequent acknowledgment late. Two synthetic lifecycle tests and 63 existing collector tests pass. This prevents an unfinished message from poisoning or prematurely satisfying a delivery gate. + +The missing VCR396 acknowledgment cannot be attributed to this issue without its raw events. Historical outcomes remain unchanged. This correction concerns prospective measurement, not advisor efficacy or native Desktop delivery. diff --git a/scripts/pilot_cli_core.py b/scripts/pilot_cli_core.py index 4b27437..e08a91b 100644 --- a/scripts/pilot_cli_core.py +++ b/scripts/pilot_cli_core.py @@ -193,6 +193,9 @@ def extract_event(event: dict[str, Any]) -> tuple[str | None, str | None, str | if normalized in {"error", "reasoning"}: return None, None, None if normalized in {"agent_message", "assistant_message", "message"}: + # Draft lifecycle events do not prove delivery of the final message. + if event_type in {"item.started", "item.updated"}: + return None, None, None return "assistant", text_value, None tool_types = { "command_execution", "function_call", "tool_call", "mcp_tool_call", diff --git a/tests/test_pilot_assistant_lifecycle.py b/tests/test_pilot_assistant_lifecycle.py new file mode 100644 index 0000000..3acc0d8 --- /dev/null +++ b/tests/test_pilot_assistant_lifecycle.py @@ -0,0 +1,29 @@ +from __future__ import annotations +import json +from pathlib import Path +import sys +import unittest +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import pilot_cli_core as core + +class AssistantLifecycleTests(unittest.TestCase): + def test_draft_events_cannot_establish_or_poison_delivery(self): + for event_type in ("item.started", "item.updated"): + for text in (None, "", "JevCompass instructions received"): + event = {"type": event_type, "item": {"type": "agent_message", "text": text}} + self.assertEqual(core.extract_event(event), (None, None, None)) + complete = {"type": "item.completed", "item": {"type": "agent_message", "text": "ack"}} + self.assertEqual(core.extract_event(complete), ("assistant", "ack", None)) + + def test_tool_before_completed_message_remains_before_delivery(self): + events = [ + {"type": "item.started", "item": {"type": "agent_message", "text": "ack"}}, + {"type": "item.started", "item": {"type": "command_execution", "command": "python --version"}}, + {"type": "item.completed", "item": {"type": "agent_message", "text": "ack"}}, + ] + kinds = [core.extract_event(event)[0] for event in events] + self.assertEqual(kinds, [None, "tool", "assistant"]) + +if __name__ == "__main__": + unittest.main()