From 30fe9c0222a01db8ca628cbf185468988d8d4be2 Mon Sep 17 00:00:00 2001 From: William Reade Date: Wed, 4 Nov 2026 16:38:26 +0000 Subject: [PATCH] fix: read claude's JSON output as an event list too [ENG-9655] With verbose on, `claude -p --output-format json` prints a list of the session's events (system/init, assistant, rate_limit_event, result) rather than the result object alone. The default command loads user settings, so `"verbose": true` there is enough. envelope() discarded the list, so every default-command call raised InferenceError and assess.py and distill.py failed. It now takes the last "result" event from a list, and still reads the object form. The envelope tests run against both shapes. Checked live on 2.1.310 with and without --verbose: main fails on the verbose output, this reads it. Co-Authored-By: Claude Opus 5.5 --- .../skills/guardrails/scripts/_inference.py | 9 ++++ tests/test_inference.py | 44 +++++++++++++------ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/plugins/stacklet/skills/guardrails/scripts/_inference.py b/plugins/stacklet/skills/guardrails/scripts/_inference.py index 99ade16..d37894e 100644 --- a/plugins/stacklet/skills/guardrails/scripts/_inference.py +++ b/plugins/stacklet/skills/guardrails/scripts/_inference.py @@ -99,6 +99,15 @@ def envelope(stdout): env = json.loads(stdout.decode("utf-8")) except ValueError: # UnicodeDecodeError too return None + # verbose (the flag, or `"verbose": true` in the user settings the call loads) prints the + # session's events as a list instead, the envelope last among them as the "result" event + if isinstance(env, list): + results = ( + e + for e in reversed(env) + if isinstance(e, dict) and e.get("type") == "result" + ) + env = next(results, None) return env if isinstance(env, dict) else None diff --git a/tests/test_inference.py b/tests/test_inference.py index ffacb40..958c5fa 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -77,15 +77,27 @@ def test_an_invalid_command_is_refused(env, bad): _inference.command("assess") -def test_the_default_command_s_envelope_gives_answer_tokens_and_cost(tmp_path): +# what `--output-format json` prints, given the envelope as a Python expression: the envelope alone, +# or (verbose, as a user setting can make it) a list of the session's events ending in it +SHAPES = { + "object": "{}", + "events": "[{{'type': 'system', 'subtype': 'init'}}, {{'type': 'assistant'}}," + " {{'type': 'rate_limit_event'}}, {}]", +} +shapes = pytest.mark.parametrize("shape", SHAPES.values(), ids=SHAPES.keys()) + + +@shapes +def test_the_default_command_s_envelope_gives_answer_tokens_and_cost(tmp_path, shape): # stands in for claude: whatever the arguments, the envelope `--output-format json` gives + envelope = ( + "{'type': 'result', 'is_error': False, 'result': ' the answer '," + " 'total_cost_usd': 0.02, 'usage': {'input_tokens': 9, 'cache_creation_input_tokens': 900," + " 'cache_read_input_tokens': 91, 'output_tokens': 40}}" + ) exe = stub( tmp_path, - "import json, sys\n" - "sys.stdin.read()\n" - "print(json.dumps({'type': 'result', 'is_error': False, 'result': ' the answer '," - " 'total_cost_usd': 0.02, 'usage': {'input_tokens': 9, 'cache_creation_input_tokens': 900," - " 'cache_read_input_tokens': 91, 'output_tokens': 40}}))\n", + f"import json, sys\nsys.stdin.read()\nprint(json.dumps({shape.format(envelope)}))\n", ) text, stats = _inference.run("q", claude_at(exe)) assert text == "the answer" @@ -94,7 +106,7 @@ def test_the_default_command_s_envelope_gives_answer_tokens_and_cost(tmp_path): assert stats["cost_usd"] == 0.02 # a custom command: its output is the answer, and only time is known text, stats = _inference.run("q", _inference.Command([str(exe)], custom=True)) - assert text.startswith('{"type": "result"') + assert text.startswith(shape[0]) assert set(stats) == {"seconds"} @@ -112,13 +124,15 @@ def test_an_envelope_reporting_an_error_is_one(tmp_path): ) -def test_a_failing_default_command_says_why_from_its_envelope(tmp_path): +@shapes +def test_a_failing_default_command_says_why_from_its_envelope(tmp_path, shape): # in JSON mode claude reports a failure on stdout, in the envelope, not on stderr + envelope = ( + "{'type': 'result', 'is_error': True, 'result': 'API Error: 529 overloaded'}" + ) exe = stub( tmp_path, - "import json, sys\n" - "print(json.dumps({'is_error': True, 'result': 'API Error: 529 overloaded'}))\n" - "sys.exit(1)\n", + f"import json, sys\nprint(json.dumps({shape.format(envelope)}))\nsys.exit(1)\n", ) with pytest.raises( _inference.InferenceError, match=r"failed \(exit 1\): API Error: 529 overloaded" @@ -175,8 +189,12 @@ def test_odd_usage_values_count_nothing_rather_than_fail(tmp_path, envelope): ) -def test_no_envelope_from_the_default_command_is_an_error(tmp_path): - exe = stub(tmp_path, "print('just text')\n") +@pytest.mark.parametrize( + "output", + ["just text", "[]", '[{"type": "system"}, {"type": "assistant"}]', '"a string"'], +) +def test_no_envelope_from_the_default_command_is_an_error(tmp_path, output): + exe = stub(tmp_path, f"print({output!r})\n") with pytest.raises(_inference.InferenceError, match="wrote no result envelope"): _inference.run( "q",