diff --git a/tee_gateway/controllers/chat_controller.py b/tee_gateway/controllers/chat_controller.py index 6084bee..7e3de4b 100644 --- a/tee_gateway/controllers/chat_controller.py +++ b/tee_gateway/controllers/chat_controller.py @@ -31,6 +31,7 @@ extract_usage, non_streaming_invoke_kwargs, validate_attachments, + validate_reasoning_effort, AttachmentValidationError, canonical_user_content, ) @@ -113,6 +114,10 @@ def create_chat_completion(body): validate_resolution(chat_request.model, chat_request.resolution) except ValueError as e: return {"error": "Invalid resolution", "message": str(e)}, 400 + try: + validate_reasoning_effort(chat_request.reasoning_effort) + except ValueError as e: + return {"error": "Invalid reasoning_effort", "message": str(e)}, 400 try: validate_aspect_ratio( chat_request.model, chat_request.aspect_ratio, chat_request.resolution @@ -311,6 +316,7 @@ def _create_non_streaming_response( force_responses_api=_needs_responses_api_for_tools( provider, cfg, tools_list ), + reasoning_effort=chat_request.reasoning_effort, ) # Bind user tools and/or the native web search tool if requested. @@ -500,6 +506,7 @@ def _create_streaming_response( force_responses_api=_needs_responses_api_for_tools( provider, cfg, tools_list ), + reasoning_effort=chat_request.reasoning_effort, ) # Bind user tools and/or the native web search tool if requested. @@ -992,6 +999,8 @@ def _chat_request_to_dict(chat_request: CreateChatCompletionRequest) -> dict: d["aspect_ratio"] = chat_request.aspect_ratio if chat_request.resolution: d["resolution"] = chat_request.resolution + if chat_request.reasoning_effort is not None: + d["reasoning_effort"] = chat_request.reasoning_effort return d @@ -1017,6 +1026,7 @@ def _parse_chat_request(chat_request_dict: dict) -> CreateChatCompletionRequest: web_search=chat_request_dict.get("web_search", False), aspect_ratio=chat_request_dict.get("aspect_ratio"), resolution=chat_request_dict.get("resolution"), + reasoning_effort=chat_request_dict.get("reasoning_effort"), ) diff --git a/tee_gateway/llm_backend.py b/tee_gateway/llm_backend.py index 9853971..25a228e 100644 --- a/tee_gateway/llm_backend.py +++ b/tee_gateway/llm_backend.py @@ -218,11 +218,13 @@ def get_chat_model_cached( temperature: float, max_tokens: int, force_responses_api: bool = False, + reasoning_effort: Optional[str] = None, ): """Get cached chat model instance using the injected ProviderConfig. - Models are cached by (model, temperature, max_tokens, force_responses_api) - tuple. Cache is cleared by set_provider_config() after key injection. + Models are cached by (model, temperature, max_tokens, force_responses_api, + reasoning_effort) tuple. Cache is cleared by set_provider_config() after + key injection. Web search needs nothing here: it is a plain function tool the gateway binds and executes itself (see web_search.py), not a provider feature that has to @@ -233,6 +235,14 @@ def get_chat_model_cached( its default ``reasoning_effort`` on Chat Completions, so the chat controller sets this flag for those models when tools are bound. Ignored by non-OpenAI providers. + + ``reasoning_effort`` is the request's own field (see + ``REASONING_EFFORTS``), translated once per provider rather than per + model: the OpenAI-compatible providers take it as-is, except ModelArk's + ``"none"`` (its documented off switch is ``thinking: disabled``) and + OpenRouter (its ``reasoning`` object). ``None`` sends nothing, leaving the + provider's default. Anthropic and Google don't translate it yet and ignore + it. A value the model doesn't support is the provider's 400 to give. """ config = _provider_config if config is None: @@ -297,6 +307,8 @@ def get_chat_model_cached( # model rejects function tools + its default reasoning_effort on Chat # Completions (gpt-5.6 family). openai_kwargs: dict[str, Any] = {} + if reasoning_effort is not None: + openai_kwargs["reasoning_effort"] = reasoning_effort if force_responses_api: openai_kwargs["use_responses_api"] = True openai_kwargs["output_version"] = "responses/v1" @@ -343,6 +355,7 @@ def get_chat_model_cached( temperature=effective_temp, max_tokens=max_tokens, http_client=xai_http_client, + reasoning_effort=reasoning_effort, streaming=True, stream_usage=True, ) @@ -361,6 +374,14 @@ def get_chat_model_cached( http_client=bytedance_http_client, api_key=SecretStr(config.bytedance_api_key), base_url=BYTEDANCE_BASE_URL, + # ModelArk rejects reasoning_effort alongside thinking: disabled, + # so "none" is sent as the switch alone. + reasoning_effort=(reasoning_effort if reasoning_effort != "none" else None), + extra_body=( + {"thinking": {"type": "disabled"}} + if reasoning_effort == "none" + else None + ), streaming=True, stream_usage=True, ) # type: ignore [call-arg] @@ -384,7 +405,12 @@ def get_chat_model_cached( {"order": [cfg.openrouter_provider], "allow_fallbacks": False} if cfg.openrouter_provider else {"sort": "price"} - ) + ), + **( + {"reasoning": {"effort": reasoning_effort}} + if reasoning_effort is not None + else {} + ), }, streaming=True, stream_usage=True, @@ -404,6 +430,7 @@ def get_chat_model_cached( http_client=zai_http_client, api_key=SecretStr(config.zai_api_key), base_url=ZAI_BASE_URL, + reasoning_effort=reasoning_effort, streaming=True, stream_usage=True, ) # type: ignore [call-arg] @@ -422,6 +449,7 @@ def get_chat_model_cached( http_client=abliteration_http_client, api_key=SecretStr(config.abliteration_api_key), base_url=ABLITERATION_BASE_URL, + reasoning_effort=reasoning_effort, streaming=True, stream_usage=True, ) # type: ignore [call-arg] @@ -430,6 +458,25 @@ def get_chat_model_cached( raise ValueError(f"Unsupported provider: {provider}") +# Values a request's ``reasoning_effort`` may take: the union of what the +# providers we translate it for accept. Which of them a given model supports is +# the provider's call, not a list kept here. +REASONING_EFFORTS = frozenset( + {"none", "minimal", "low", "medium", "high", "xhigh", "max"} +) + + +def validate_reasoning_effort(value: Any) -> None: + """Raise ``ValueError`` unless ``value`` is absent or a known effort.""" + if value is None: + return + if not isinstance(value, str) or value not in REASONING_EFFORTS: + raise ValueError( + f"Unsupported reasoning_effort {value!r}; " + f"supported: {', '.join(sorted(REASONING_EFFORTS))}" + ) + + def _parse_data_uri(uri: str) -> Optional[tuple[str, str]]: """Parse a ``data:;base64,`` URI into ``(mime_type, base64_data)``. diff --git a/tee_gateway/models/create_chat_completion_request.py b/tee_gateway/models/create_chat_completion_request.py index d4ace2f..a81fecd 100644 --- a/tee_gateway/models/create_chat_completion_request.py +++ b/tee_gateway/models/create_chat_completion_request.py @@ -6,7 +6,7 @@ def __init__( messages=None, model=None, store=False, - reasoning_effort="medium", + reasoning_effort=None, metadata=None, frequency_penalty=0, logit_bias=None, diff --git a/tee_gateway/openapi/openapi.yaml b/tee_gateway/openapi/openapi.yaml index 3ed3b31..7eb63ac 100644 --- a/tee_gateway/openapi/openapi.yaml +++ b/tee_gateway/openapi/openapi.yaml @@ -2790,7 +2790,6 @@ components: } CreateChatCompletionRequest: example: - reasoning_effort: medium top_logprobs: 2 metadata: key: metadata @@ -2923,16 +2922,22 @@ components: title: store type: boolean reasoning_effort: - default: medium - description: "**o1 models only** \n\nConstrains effort on reasoning for\ - \ \n[reasoning models](https://platform.openai.com/docs/guides/reasoning).\n\ - Currently supported values are `low`, `medium`, and `high`. Reducing\n\ - reasoning effort can result in faster responses and fewer tokens used\n\ - on reasoning in a response.\n" - enum: + description: | + How much the model reasons before answering. Omitted means the + provider's default. Passed to OpenAI, xAI, ByteDance ModelArk, + OpenRouter, Z.ai and abliteration.ai in each one's own format + (`none` turns reasoning off); ignored by Anthropic and Google. Which + values a model accepts is the provider's to decide, and a value it + rejects comes back as the provider's 4xx. Part of the signed + request hash when sent. + enum: + - none + - minimal - low - medium - high + - xhigh + - max title: reasoning_effort type: string metadata: diff --git a/tee_gateway/test/test_chat_controller.py b/tee_gateway/test/test_chat_controller.py index 7568cce..304978b 100644 --- a/tee_gateway/test/test_chat_controller.py +++ b/tee_gateway/test/test_chat_controller.py @@ -193,6 +193,74 @@ def test_auto_binds_nothing( self.assertIn("choices", result) +class TestReasoningEffortRequests(unittest.TestCase): + """``reasoning_effort`` is validated, forwarded, and signed when sent.""" + + def _request(self, **extra): + body = { + "model": "deepseek-v4-flash", + "messages": [{"role": "user", "content": "hi"}], + "stream": False, + } + body.update(extra) + return body + + def test_none_passes_schema_validation(self): + resp = ( + _build_app() + .test_client() + .post( + "/v1/chat/completions", + data=json.dumps(self._request(reasoning_effort="none")), + content_type="application/json", + headers={"Authorization": "Bearer test"}, + ) + ) + self.assertNotEqual(400, resp.status_code, resp.data.decode("utf-8")) + + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_unknown_effort_is_a_400(self, mock_connexion): + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = self._request( + reasoning_effort="extreme" + ) + + result, status = create_chat_completion(None) + + self.assertEqual(400, status) + self.assertEqual("Invalid reasoning_effort", result["error"]) + + @patch("tee_gateway.controllers.chat_controller.get_tee_keys") + @patch("tee_gateway.controllers.chat_controller.get_chat_model_cached") + @patch("tee_gateway.controllers.chat_controller.connexion") + def test_effort_reaches_the_model_factory( + self, mock_connexion, mock_get_model, mock_get_tee_keys + ): + mock_connexion.request.is_json = True + mock_connexion.request.get_json.return_value = self._request( + reasoning_effort="none" + ) + mock_get_model.return_value = _mock_image_model() + mock_get_tee_keys.return_value = _mock_tee_keys() + + create_chat_completion(None) + + self.assertEqual("none", mock_get_model.call_args.kwargs["reasoning_effort"]) + + def test_signed_hash_covers_effort_only_when_sent(self): + from tee_gateway.controllers.chat_controller import ( + _chat_request_to_dict, + _parse_chat_request, + ) + + plain = _chat_request_to_dict(_parse_chat_request(self._request())) + self.assertNotIn("reasoning_effort", plain) + sent = _chat_request_to_dict( + _parse_chat_request(self._request(reasoning_effort="low")) + ) + self.assertEqual("low", sent["reasoning_effort"]) + + def _mock_image_model() -> Mock: """A LangChain chat model stand-in that returns one inline image.""" response = Mock() diff --git a/tee_gateway/test/test_tee_core.py b/tee_gateway/test/test_tee_core.py index 7745116..9577971 100644 --- a/tee_gateway/test/test_tee_core.py +++ b/tee_gateway/test/test_tee_core.py @@ -1072,3 +1072,53 @@ def test_deepseek_v4_pro_pinned_to_phala(self): def test_unpinned_model_sorts_by_price(self): model = get_chat_model_cached("hy3", 0.0, 1024) self.assertEqual(model.extra_body, {"provider": {"sort": "price"}}) + + +class TestReasoningEffortTranslation(unittest.TestCase): + """One translation per provider; omitted means nothing is sent.""" + + _saved_config: Optional[ProviderConfig] = None + + @classmethod + def setUpClass(cls) -> None: + cls._saved_config = get_provider_config() + set_provider_config( + ProviderConfig( + openai_api_key="test-openai-key", + bytedance_api_key="test-bytedance-key", + openrouter_api_key="test-openrouter-key", + ) + ) + + @classmethod + def tearDownClass(cls) -> None: + if cls._saved_config is not None: + set_provider_config(cls._saved_config) + get_chat_model_cached.cache_clear() + + def test_omitted_sends_nothing(self): + model = get_chat_model_cached("deepseek-v4-flash", 0.0, 1024) + self.assertIsNone(model.reasoning_effort) + self.assertIsNone(model.extra_body) + + def test_modelark_none_is_the_thinking_switch_alone(self): + model = get_chat_model_cached("deepseek-v4-flash", 0.0, 1024, False, "none") + self.assertEqual(model.extra_body, {"thinking": {"type": "disabled"}}) + self.assertNotIn("reasoning_effort", model._default_params) + + def test_modelark_level_is_sent_as_is(self): + model = get_chat_model_cached("deepseek-v4-flash", 0.0, 1024, False, "max") + self.assertEqual(model._default_params["reasoning_effort"], "max") + self.assertIsNone(model.extra_body) + + def test_openrouter_uses_its_reasoning_object(self): + model = get_chat_model_cached("deepseek-v4-pro", 0.0, 1024, False, "none") + self.assertEqual(model.extra_body["reasoning"], {"effort": "none"}) + self.assertEqual( + model.extra_body["provider"], + {"order": ["phala"], "allow_fallbacks": False}, + ) + + def test_openai_takes_it_as_is(self): + model = get_chat_model_cached("gpt-5.4", 0.0, 1024, False, "low") + self.assertEqual(model._default_params["reasoning_effort"], "low")