Skip to content

fix: read claude's JSON output as an event list too (ENG-9655) #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fwereade wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: read claude's JSON output as an event list too (ENG-9655) #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions plugins/stacklet/skills/guardrails/scripts/_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,15 @@ def envelope(stdout):
env = json.loads(stdout.decode("utf-8"))
except ValueError: # UnicodeDecodeError too
return None
# verbose (the flag, or `"verbose": true` in the user settings the call loads) prints the
# session's events as a list instead, the envelope last among them as the "result" event
if isinstance(env, list):
results = (
e
for e in reversed(env)
if isinstance(e, dict) and e.get("type") == "result"
)
env = next(results, None)
return env if isinstance(env, dict) else None


Expand Down
44 changes: 31 additions & 13 deletions tests/test_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,15 +77,27 @@ def test_an_invalid_command_is_refused(env, bad):
_inference.command("assess")


def test_the_default_command_s_envelope_gives_answer_tokens_and_cost(tmp_path):
# what `--output-format json` prints, given the envelope as a Python expression: the envelope alone,
# or (verbose, as a user setting can make it) a list of the session's events ending in it
SHAPES = {
"object": "{}",
"events": "[{{'type': 'system', 'subtype': 'init'}}, {{'type': 'assistant'}},"
" {{'type': 'rate_limit_event'}}, {}]",
}
shapes = pytest.mark.parametrize("shape", SHAPES.values(), ids=SHAPES.keys())


@shapes
def test_the_default_command_s_envelope_gives_answer_tokens_and_cost(tmp_path, shape):
# stands in for claude: whatever the arguments, the envelope `--output-format json` gives
envelope = (
"{'type': 'result', 'is_error': False, 'result': ' the answer ',"
" 'total_cost_usd': 0.02, 'usage': {'input_tokens': 9, 'cache_creation_input_tokens': 900,"
" 'cache_read_input_tokens': 91, 'output_tokens': 40}}"
)
exe = stub(
tmp_path,
"import json, sys\n"
"sys.stdin.read()\n"
"print(json.dumps({'type': 'result', 'is_error': False, 'result': ' the answer ',"
" 'total_cost_usd': 0.02, 'usage': {'input_tokens': 9, 'cache_creation_input_tokens': 900,"
" 'cache_read_input_tokens': 91, 'output_tokens': 40}}))\n",
f"import json, sys\nsys.stdin.read()\nprint(json.dumps({shape.format(envelope)}))\n",
)
text, stats = _inference.run("q", claude_at(exe))
assert text == "the answer"
Expand All @@ -94,7 +106,7 @@ def test_the_default_command_s_envelope_gives_answer_tokens_and_cost(tmp_path):
assert stats["cost_usd"] == 0.02
# a custom command: its output is the answer, and only time is known
text, stats = _inference.run("q", _inference.Command([str(exe)], custom=True))
assert text.startswith('{"type": "result"')
assert text.startswith(shape[0])
assert set(stats) == {"seconds"}


Expand All @@ -112,13 +124,15 @@ def test_an_envelope_reporting_an_error_is_one(tmp_path):
)


def test_a_failing_default_command_says_why_from_its_envelope(tmp_path):
@shapes
def test_a_failing_default_command_says_why_from_its_envelope(tmp_path, shape):
# in JSON mode claude reports a failure on stdout, in the envelope, not on stderr
envelope = (
"{'type': 'result', 'is_error': True, 'result': 'API Error: 529 overloaded'}"
)
exe = stub(
tmp_path,
"import json, sys\n"
"print(json.dumps({'is_error': True, 'result': 'API Error: 529 overloaded'}))\n"
"sys.exit(1)\n",
f"import json, sys\nprint(json.dumps({shape.format(envelope)}))\nsys.exit(1)\n",
)
with pytest.raises(
_inference.InferenceError, match=r"failed \(exit 1\): API Error: 529 overloaded"
Expand Down Expand Up @@ -175,8 +189,12 @@ def test_odd_usage_values_count_nothing_rather_than_fail(tmp_path, envelope):
)


def test_no_envelope_from_the_default_command_is_an_error(tmp_path):
exe = stub(tmp_path, "print('just text')\n")
@pytest.mark.parametrize(
"output",
["just text", "[]", '[{"type": "system"}, {"type": "assistant"}]', '"a string"'],
)
def test_no_envelope_from_the_default_command_is_an_error(tmp_path, output):
exe = stub(tmp_path, f"print({output!r})\n")
with pytest.raises(_inference.InferenceError, match="wrote no result envelope"):
_inference.run(
"q",
Expand Down
Loading