diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml index a3e1cd8b..8a4ab01f 100644 --- a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml @@ -1,8 +1,9 @@ interactions: - request: - body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The - customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is - the sentiment positive?"}]}' + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"Look + up the customer feedback."}]},{"type":"function_call","call_id":"feedback","name":"get_feedback","arguments":"{}"},{"type":"function_call_output","call_id":"feedback","output":[{"type":"input_text","text":"The + customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":[{"type":"input_text","text":"Is + the sentiment positive?"}]}]}' headers: accept: - application/json @@ -11,7 +12,7 @@ interactions: connection: - keep-alive content-length: - - '228' + - '470' content-type: - application/json host: @@ -40,7 +41,7 @@ interactions: uri: https://api.openai.com/v1/decisions response: body: - string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":162,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":162}}' + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":193,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":193}}' headers: access-control-allow-origin: - '*' @@ -52,7 +53,7 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a50979c08bf436b2-YYZ + - a509b770ccf8c0f6-YYZ connection: - keep-alive content-length: @@ -60,13 +61,13 @@ interactions: content-type: - application/json date: - - Mon, 26 Oct 2026 12:31:01 GMT + - Mon, 26 Oct 2026 13:13:08 GMT openai-processing-ms: - - '41' + - '153' server: - cloudflare server-timing-total: - - '41.175' + - '153.470' strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: @@ -77,20 +78,8 @@ interactions: - nosniff x-openai-proxy-wasm: - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '180000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '179999838' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s x-request-id: - - req_d23510e8e57c4086b8e639b76fedd7de + - req_a794b137a7f7449eb1f000c360d6a7ff status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml index 0ed3f6be..60729bb6 100644 --- a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml @@ -1,8 +1,9 @@ interactions: - request: - body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The - customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is - the sentiment positive?"}]}' + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"Look + up the customer feedback."}]},{"type":"function_call","call_id":"feedback","name":"get_feedback","arguments":"{}"},{"type":"function_call_output","call_id":"feedback","output":[{"type":"input_text","text":"The + customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":[{"type":"input_text","text":"Is + the sentiment positive?"}]}]}' headers: accept: - application/json @@ -11,7 +12,7 @@ interactions: connection: - keep-alive content-length: - - '228' + - '470' content-type: - application/json host: @@ -40,7 +41,7 @@ interactions: uri: https://api.openai.com/v1/decisions response: body: - string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":162,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":162}}' + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":193,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":193}}' headers: access-control-allow-origin: - '*' @@ -52,7 +53,7 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a50979bf1f69a217-YYZ + - a509b76f5d80711b-YYZ connection: - keep-alive content-length: @@ -60,13 +61,13 @@ interactions: content-type: - application/json date: - - Mon, 26 Oct 2026 12:31:01 GMT + - Mon, 26 Oct 2026 13:13:07 GMT openai-processing-ms: - - '37' + - '47' server: - cloudflare server-timing-total: - - '37.446' + - '46.996' strict-transport-security: - max-age=31536000; includeSubDomains; preload transfer-encoding: @@ -84,13 +85,13 @@ interactions: x-ratelimit-remaining-requests: - '29999' x-ratelimit-remaining-tokens: - - '179999838' + - '179999805' x-ratelimit-reset-requests: - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_2a5267c20681464eabd0985a408d2b4e + - req_24e6b323ed074579a34269a3123adbf0 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 5e211c67..c6402ffa 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -240,26 +240,41 @@ async def test_openai_decisions_metrics(memory_logger, is_async): @pytest.mark.asyncio @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) async def test_openai_decisions_iterable_input(memory_logger, is_async): - if not version_satisfies(openai.__version__, ">=3.26.0"): - pytest.skip("Decisions requires OpenAI >=3.26.0") - - messages = [{"role": "user", "content": [{"type": "input_text", "text": DECISIONS_INPUT}]}] + if not version_satisfies(openai.__version__, ">=3.37.0"): + pytest.skip("Iterable Decisions instructions require OpenAI >=3.37.0") + + instructions = [{"type": "input_text", "text": "Is the sentiment positive?"}] + output = [{"type": "input_text", "text": DECISIONS_INPUT}] + messages = [ + {"role": "user", "content": [{"type": "input_text", "text": "Look up the customer feedback."}]}, + {"type": "function_call", "call_id": "feedback", "name": "get_feedback", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "feedback", "output": output}, + ] content = iter(messages[0]["content"]) - request_messages = [{**messages[0], "content": content}] + instruction_parts = (part for part in instructions) + output_parts = (part for part in output) + questions = [{"type": "predicate", "name": "positive", "instructions": instruction_parts}] + request_messages = [{**messages[0], "content": content}, messages[1], {**messages[2], "output": output_parts}] client = wrap_openai(_client(is_async)) response = await _call( is_async, client.decisions.create, model=DECISIONS_MODEL, input=iter(request_messages), - questions=DECISIONS_QUESTIONS[:1], + questions=iter(questions), ) + assert request_messages[0]["content"] is content + assert questions[0]["instructions"] is instruction_parts + assert request_messages[2]["output"] is output_parts spans = memory_logger.pop() assert len(spans) == 1 - assert request_messages[0]["content"] is content - assert spans[0]["input"]["state"] == messages - assert spans[0]["output"]["answers"] == [ + span = spans[0] + assert span["input"] == { + "state": messages, + "questions": [{"id": "positive", "type": "boolean", "instructions": instructions}], + } + assert span["output"]["answers"] == [ {"id": "positive", "type": "boolean", "probability": response.answers[0].probability} ] diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index b6ffa990..38c83570 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -2197,7 +2197,10 @@ def _materialize_decision_items(values: Any, nested_fields: tuple[str, ...]) -> def _decisions_span_params(params: dict[str, Any]) -> dict[str, Any]: ret = params.pop("span_info", {}) - for field, nested_fields in (("questions", ("choices", "levels")), ("input", ("content",))): + for field, nested_fields in ( + ("questions", ("choices", "levels", "instructions")), + ("input", ("content", "output")), + ): if field in params: params[field] = _materialize_decision_items(params[field], nested_fields) return merge_dicts(