diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts index dbbfe63253..60a0212a24 100644 --- a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts +++ b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts @@ -19,6 +19,22 @@ export function getMaxTokens(request: GatewayRequest) { return request.body.max_completion_tokens ?? request.body.max_tokens ?? null; } +export function getReasoningEffort(request: GatewayRequest) { + if (request.kind === 'messages') { + return request.body.output_config?.effort ?? null; + } + if (request.kind === 'responses') { + return request.body.reasoning?.effort ?? null; + } + return request.body.reasoning?.effort ?? request.body.reasoning_effort ?? null; +} + +export function getReasoningEffortTimeoutSuggestion(effort: string | null): string { + return effort === 'xhigh' || effort === 'max' + ? ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts.` + : ''; +} + export function hasMiddleOutTransform(request: GatewayRequest) { return ( (request.kind === 'chat_completions' && request.body.transforms?.includes('middle-out')) || diff --git a/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts b/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts index cf84d5ad03..f1bea7edb3 100644 --- a/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts +++ b/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts @@ -8,6 +8,7 @@ import { applyProviderSpecificLogic } from '@/lib/ai-gateway/providers/apply-pro import type { GetProviderProviderResult } from '@/lib/ai-gateway/providers/get-provider'; import { isValidOpenRouterModelId } from '@/lib/ai-gateway/providers/gateway-models-cache'; import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; +import { getReasoningEffort } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import { upstreamRequest } from '@/lib/ai-gateway/providers/upstream-request'; import type { FraudDetectionHeaders } from '@/lib/utils'; @@ -92,6 +93,7 @@ export async function sendUpstreamAttempt({ provider: providerContext.provider, signal, vercelRequestId, + reasoningEffort: getReasoningEffort(request), }); if (result.type === 'error') return result; diff --git a/apps/web/src/lib/ai-gateway/providers/upstream-request.ts b/apps/web/src/lib/ai-gateway/providers/upstream-request.ts index bc362f573d..b26cb10032 100644 --- a/apps/web/src/lib/ai-gateway/providers/upstream-request.ts +++ b/apps/web/src/lib/ai-gateway/providers/upstream-request.ts @@ -9,6 +9,7 @@ import type { GatewayMessagesRequest, } from '@/lib/ai-gateway/providers/openrouter/types'; import { ATTRIBUTION_HEADERS } from '@/lib/ai-gateway/providers/openrouter/attribution-headers'; +import { getReasoningEffortTimeoutSuggestion } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import type { GatewayChatApiKind, Provider } from '@/lib/ai-gateway/providers/types'; import { after, NextResponse } from 'next/server'; import { ProxyErrorType } from '@/lib/proxy-error-types'; @@ -172,14 +173,15 @@ function clientDisconnectResponse(vercelRequestId: string | null | undefined) { function upstreamFetchFailureResponse( failureFamily: UpstreamFetchFailureFamily, - vercelRequestId: string | null | undefined + vercelRequestId: string | null | undefined, + reasoningEffort: string | null ) { const error = withRequestId( failureFamily === 'request_timeout' || failureFamily === 'headers_timeout' || failureFamily === 'connect_timeout' || failureFamily === 'read_timeout' - ? 'The upstream provider did not send response headers before the gateway timeout.' + ? `The upstream provider did not send response headers before the gateway timeout.${getReasoningEffortTimeoutSuggestion(reasoningEffort)}` : 'The upstream provider closed the connection before sending a response.', vercelRequestId ); @@ -203,6 +205,7 @@ export async function upstreamRequest({ provider, signal, vercelRequestId, + reasoningEffort, }: { chatApi: GatewayChatApiKind; search: string; @@ -213,6 +216,7 @@ export async function upstreamRequest({ signal?: AbortSignal; /** Incoming `x-vercel-id`, used to correlate failures with the platform logs. */ vercelRequestId?: string | null; + reasoningEffort: string | null; }): Promise<{ type: 'success'; response: Response } | { type: 'error'; response: NextResponse }> { const headers = new Headers(); for (const [key, value] of Object.entries(ATTRIBUTION_HEADERS)) { @@ -309,7 +313,11 @@ export async function upstreamRequest({ type: 'error', response: causedByClientDisconnect ? clientDisconnectResponse(vercelRequestId) - : upstreamFetchFailureResponse(failureFamily ?? 'unknown', vercelRequestId), + : upstreamFetchFailureResponse( + failureFamily ?? 'unknown', + vercelRequestId, + reasoningEffort + ), }; } } diff --git a/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts b/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts index e6e22d49b0..b42f2438f8 100644 --- a/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts +++ b/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts @@ -14,6 +14,7 @@ import { QWEN37_PLUS_MODEL_ID } from '@/lib/ai-gateway/custom-pricing'; import { KILO_ORGANIZATION_ID } from '@/lib/organizations/constants'; import { logExceptInTest } from '@/lib/utils.server'; import { ReasoningDetailsTransform } from '@/lib/ai-gateway/providers/types'; +import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; jest.mock('next/server', () => ({ ...(jest.requireActual('next/server') as Record), @@ -141,6 +142,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(503); @@ -151,12 +153,32 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { }); }); + test.each(['high', 'medium', 'low', 'none', null])( + 'omits effort advice for effort %s', + async reasoningEffort => { + const result = await rewrite({ + response: failingResponse('application/json', 'TimeoutError'), + removeCost: true, + capture: null, + vercelRequestId: null, + reasoningEffort, + }); + + expect(await result.json()).toEqual({ + error: 'The upstream provider timed out while sending the response.', + error_type: 'timeout', + message: 'The upstream provider timed out while sending the response.', + }); + } + ); + test('includes the vercel request id only in the JSON read error message', async () => { const result = await rewrite({ response: failingResponse('application/json', 'ResponseAborted'), removeCost: true, capture: null, vercelRequestId: 'iad1::iad1::request-id', + reasoningEffort: null, }); expect(result.status).toBe(503); @@ -175,6 +197,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: 'iad1::iad1::request-id', + reasoningEffort: null, }); const events = dataObjects(await readOutputStream(result)) as { error: { message: string }; @@ -193,6 +216,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const events = dataObjects(await readOutputStream(result)) as { error: { message: string }; @@ -204,6 +228,91 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { }); }); +const timeoutEffortRequests = [ + { + name: 'Chat Completions nested effort', + request: { + kind: 'chat_completions', + body: { model: 'test-model', messages: [], reasoning: { effort: 'xhigh' } }, + }, + effort: 'xhigh', + }, + { + name: 'Chat Completions top-level effort', + request: { + kind: 'chat_completions', + body: { model: 'test-model', messages: [], reasoning_effort: 'max' }, + }, + effort: 'max', + }, + { + name: 'Responses', + request: { + kind: 'responses', + body: { model: 'test-model', input: 'Hello', reasoning: { effort: 'xhigh' } }, + }, + effort: 'xhigh', + }, + { + name: 'Messages', + request: { + kind: 'messages', + body: { + model: 'test-model', + messages: [], + max_tokens: 1024, + output_config: { effort: 'max' }, + }, + }, + effort: 'max', + }, +] satisfies { name: string; request: GatewayRequest; effort: string }[]; + +describe.each(timeoutEffortRequests)('$name timeout effort guidance', ({ request, effort }) => { + test.each([ + ['application/json', 'TimeoutError', 'timeout'], + ['application/json', 'ResponseAborted', 'upstream_disconnect'], + ['text/event-stream', 'TimeoutError', 'timeout'], + ['text/event-stream', 'ResponseAborted', 'upstream_disconnect'], + ])('suggests lowering effort for %s %s', async (contentType, errorName, errorType) => { + const result = await rewriteModelResponse({ + response: failingResponse(contentType, errorName), + model: request.body.model ?? 'test-model', + providerId: 'openrouter', + kind: request.kind, + logging: { + user: null, + organization_id: null, + session_id: null, + vercel_request_id: 'fra1::request-id', + request, + }, + responseTransforms: null, + }); + const expectedMessage = + (errorName === 'TimeoutError' + ? 'The upstream provider timed out while sending the response.' + : 'The upstream response was interrupted while streaming. The provider may have disconnected or the request may have timed out.') + + ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts. (request id: fra1::request-id)`; + + if (contentType === 'application/json') { + expect(result.status).toBe(503); + expect(await result.json()).toEqual({ + error: expectedMessage, + error_type: errorType, + message: expectedMessage, + }); + } else { + expect(result.status).toBe(200); + expect(dataObjects(await readOutputStream(result))).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ message: expectedMessage }), + }), + ]); + } + }); +}); + describe('rewriteModelResponse_ChatCompletions', () => { describe('JSON responses', () => { test('strips upstream cost fields', async () => { @@ -225,6 +334,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const json = await result.json(); @@ -254,6 +364,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const json = await result.json(); @@ -273,6 +384,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); @@ -298,6 +410,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.ReasoningContent, }); const json = await result.json(); @@ -333,6 +446,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.ReasoningContent, }); const json = await result.json(); @@ -360,6 +474,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const json = await result.json(); @@ -395,6 +510,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const reader = result.body?.getReader(); @@ -426,6 +542,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -448,6 +565,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -473,6 +591,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.GeminiThought, }); const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ @@ -522,6 +641,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.GeminiThought, }); const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ @@ -552,6 +672,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.ReasoningContent, }); const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ @@ -583,6 +704,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -604,6 +726,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture, vercelRequestId: 'iad1::terminal-request', + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -634,6 +757,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -664,6 +788,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -682,6 +807,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); @@ -704,6 +830,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -746,6 +873,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const json = await result.json(); @@ -767,6 +895,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(500); @@ -785,6 +914,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); const events = dataObjects(sse) as Array<{ @@ -819,6 +949,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -837,6 +968,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -868,6 +1000,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -893,6 +1026,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -935,6 +1069,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const json = await result.json(); @@ -956,6 +1091,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); const [event] = dataObjects(sse) as Array<{ @@ -992,6 +1128,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -1050,6 +1187,7 @@ describe.each([ removeCost: true, capture, vercelRequestId: 'iad1::error-request', + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -1315,6 +1453,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(200); @@ -1333,6 +1472,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1351,6 +1491,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1370,6 +1511,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1389,6 +1531,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1408,6 +1551,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(503); @@ -1427,6 +1571,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const reader = result.body?.getReader(); @@ -1448,6 +1593,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const reader = result.body?.getReader(); await reader?.read(); diff --git a/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts b/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts index 765159a2aa..645b91dd70 100644 --- a/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts +++ b/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts @@ -3,6 +3,10 @@ import { isKiloExclusiveFreeModel } from '@/lib/ai-gateway/models'; import { getCustomPricing } from '@/lib/ai-gateway/custom-pricing'; import { detectToolCallArgumentErrors } from '@/lib/ai-gateway/api-request-log-errors'; import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; +import { + getReasoningEffort, + getReasoningEffortTimeoutSuggestion, +} from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import type { ProviderId, ProviderResponseTransforms } from '@/lib/ai-gateway/providers/types'; import { getOutputHeaders } from '@/lib/ai-gateway/llm-proxy-helpers'; import type { ChatCompletionChunk, OpenRouterUsage } from '@/lib/ai-gateway/processUsage.types'; @@ -49,6 +53,7 @@ export type RewriteResponseParams = { removeCost: boolean; capture: RequestLogCapture | null; vercelRequestId: string | null; + reasoningEffort: string | null; }; export type RewriteChatCompletionsResponseParams = RewriteResponseParams & { @@ -247,7 +252,8 @@ function logTerminalStreamEvent( function getResponseReadError( error: unknown, - vercelRequestId: string | null | undefined + vercelRequestId: string | null | undefined, + reasoningEffort: string | null ): ResponseReadError | null { if (typeof error !== 'object' || error === null || !('name' in error)) { return null; @@ -257,7 +263,8 @@ function getResponseReadError( return { errorType: 'upstream_disconnect', message: withRequestId( - 'The upstream response was interrupted while streaming. The provider may have disconnected or the request may have timed out.', + 'The upstream response was interrupted while streaming. The provider may have disconnected or the request may have timed out.' + + getReasoningEffortTimeoutSuggestion(reasoningEffort), vercelRequestId ), }; @@ -267,7 +274,8 @@ function getResponseReadError( return { errorType: 'timeout', message: withRequestId( - 'The upstream provider timed out while sending the response.', + 'The upstream provider timed out while sending the response.' + + getReasoningEffortTimeoutSuggestion(reasoningEffort), vercelRequestId ), }; @@ -280,12 +288,13 @@ async function readResponseText( response: Response, headers: Headers, vercelRequestId: string | null | undefined, - capture: RequestLogCapture | null + capture: RequestLogCapture | null, + reasoningEffort: string | null ): Promise<{ text: string } | { error: unknown; errorResponse: NextResponse }> { try { return { text: await response.text() }; } catch (error) { - const responseReadError = getResponseReadError(error, vercelRequestId); + const responseReadError = getResponseReadError(error, vercelRequestId, reasoningEffort); if (!responseReadError) { // Settle the capture so the after() callback awaiting it does not hang // and the request is still logged (without a response body). @@ -321,7 +330,8 @@ async function rewriteSseStream( onFinally: () => void, vercelRequestId: string | null | undefined, capture: RequestLogCapture | null, - capturedChunks: string[] | null + capturedChunks: string[] | null, + reasoningEffort: string | null ) { const decoder = new TextDecoder(); const settleReadError = (error: unknown) => @@ -368,7 +378,7 @@ async function rewriteSseStream( } } } catch (error) { - const responseReadError = getResponseReadError(error, vercelRequestId); + const responseReadError = getResponseReadError(error, vercelRequestId, reasoningEffort); if (!responseReadError) { settleReadError(error); throw error; @@ -404,6 +414,7 @@ export async function rewriteModelResponse_ChatCompletions({ response, removeCost, capture, + reasoningEffort, vercelRequestId, responseTransforms, }: RewriteChatCompletionsResponseParams) { @@ -412,7 +423,13 @@ export async function rewriteModelResponse_ChatCompletions({ if (headers.get('content-type')?.includes('application/json')) { // Read the body text once to avoid "Response body object should not be // disturbed or locked" errors that occur when `.clone().json()` fails. - const textResult = await readResponseText(response, headers, vercelRequestId, capture); + const textResult = await readResponseText( + response, + headers, + vercelRequestId, + capture, + reasoningEffort + ); if ('errorResponse' in textResult) { capture?.setReadError(textResult.error); return textResult.errorResponse; @@ -539,7 +556,8 @@ export async function rewriteModelResponse_ChatCompletions({ progress.stop, vercelRequestId, capture, - capturedChunks + capturedChunks, + reasoningEffort ); }, cancel() { @@ -591,12 +609,19 @@ export async function rewriteModelResponse_Messages({ response, removeCost, capture, + reasoningEffort, vercelRequestId, }: RewriteResponseParams) { const headers = getOutputHeaders(response); if (headers.get('content-type')?.includes('application/json')) { - const textResult = await readResponseText(response, headers, vercelRequestId, capture); + const textResult = await readResponseText( + response, + headers, + vercelRequestId, + capture, + reasoningEffort + ); if ('errorResponse' in textResult) { capture?.setReadError(textResult.error); return textResult.errorResponse; @@ -719,7 +744,8 @@ export async function rewriteModelResponse_Messages({ progress.stop, vercelRequestId, capture, - capturedChunks + capturedChunks, + reasoningEffort ); }, cancel() { @@ -747,12 +773,19 @@ export async function rewriteModelResponse_Responses({ response, removeCost, capture, + reasoningEffort, vercelRequestId, }: RewriteResponseParams) { const headers = getOutputHeaders(response); if (headers.get('content-type')?.includes('application/json')) { - const textResult = await readResponseText(response, headers, vercelRequestId, capture); + const textResult = await readResponseText( + response, + headers, + vercelRequestId, + capture, + reasoningEffort + ); if ('errorResponse' in textResult) { capture?.setReadError(textResult.error); return textResult.errorResponse; @@ -870,7 +903,8 @@ export async function rewriteModelResponse_Responses({ progress.stop, vercelRequestId, capture, - capturedChunks + capturedChunks, + reasoningEffort ); }, cancel() { @@ -905,11 +939,13 @@ export async function rewriteModelResponse({ console.debug('[rewriteModelResponse] rewriting response for %s', model); const { vercel_request_id: vercelRequestId } = logging; + const reasoningEffort = getReasoningEffort(logging.request); if (kind === 'chat_completions') { return rewriteModelResponse_ChatCompletions({ response, removeCost: requiresCostRemoval, capture, + reasoningEffort, vercelRequestId, responseTransforms, }); @@ -919,6 +955,7 @@ export async function rewriteModelResponse({ response, removeCost: requiresCostRemoval, capture, + reasoningEffort, vercelRequestId, }); } @@ -927,6 +964,7 @@ export async function rewriteModelResponse({ response, removeCost: requiresCostRemoval, capture, + reasoningEffort, vercelRequestId, }); } diff --git a/apps/web/src/tests/openrouter-request-helpers.test.ts b/apps/web/src/tests/openrouter-request-helpers.test.ts index 68f02fbc86..7b041b6658 100644 --- a/apps/web/src/tests/openrouter-request-helpers.test.ts +++ b/apps/web/src/tests/openrouter-request-helpers.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from '@jest/globals'; import { addCacheBreakpoints, + getReasoningEffort, + getReasoningEffortTimeoutSuggestion, removeChatCompletionsToolNames, } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import type { @@ -9,6 +11,81 @@ import type { } from '@/lib/ai-gateway/providers/openrouter/types'; import type OpenAI from 'openai'; +describe('getReasoningEffort', () => { + test.each([ + { options: { output_config: { effort: 'max' } }, expected: 'max' }, + { options: { output_config: { effort: 'high' } }, expected: 'high' }, + { options: { output_config: {} }, expected: null }, + { options: {}, expected: null }, + ] as const)('reads messages effort from $options as $expected', ({ options, expected }) => { + expect( + getReasoningEffort({ + kind: 'messages', + body: { model: 'test-model', messages: [], max_tokens: 1024, ...options }, + }) + ).toBe(expected); + }); + + test.each([ + { options: { reasoning: { effort: 'xhigh' } }, expected: 'xhigh' }, + { options: { reasoning: { effort: 'high' } }, expected: 'high' }, + { options: { reasoning: { effort: null } }, expected: null }, + { options: { reasoning: {} }, expected: null }, + { options: { reasoning: null }, expected: null }, + { options: {}, expected: null }, + ] as const)('reads responses effort from $options as $expected', ({ options, expected }) => { + expect( + getReasoningEffort({ + kind: 'responses', + body: { model: 'test-model', input: 'test', ...options }, + }) + ).toBe(expected); + }); + + test.each([ + { options: { reasoning: { effort: 'xhigh' } }, expected: 'xhigh' }, + { options: { reasoning_effort: 'xhigh' }, expected: 'xhigh' }, + { + options: { reasoning: { effort: 'high' }, reasoning_effort: 'xhigh' }, + expected: 'high', + }, + { + options: { reasoning: { effort: 'none' }, reasoning_effort: 'xhigh' }, + expected: 'none', + }, + { options: { reasoning: { effort: null }, reasoning_effort: 'xhigh' }, expected: 'xhigh' }, + { options: { reasoning: {}, reasoning_effort: 'high' }, expected: 'high' }, + { options: { reasoning: { effort: null }, reasoning_effort: null }, expected: null }, + { options: { reasoning: {} }, expected: null }, + { options: {}, expected: null }, + ] as const)( + 'reads chat completions effort from $options as $expected', + ({ options, expected }) => { + expect( + getReasoningEffort({ + kind: 'chat_completions', + body: { model: 'test-model', messages: [], ...options }, + }) + ).toBe(expected); + } + ); +}); + +describe('getReasoningEffortTimeoutSuggestion', () => { + test.each(['xhigh', 'max'])('suggests lowering %s effort', effort => { + expect(getReasoningEffortTimeoutSuggestion(effort)).toBe( + ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts.` + ); + }); + + test.each(['high', 'medium', 'low', 'minimal', 'none', 'XHIGH', 'MAX', 'max ', '', null])( + 'does not suggest lowering %s effort', + effort => { + expect(getReasoningEffortTimeoutSuggestion(effort)).toBe(''); + } + ); +}); + describe('removeChatCompletionsToolNames', () => { test('removes names from tool messages only', () => { const toolMessage: OpenAI.ChatCompletionToolMessageParam & { name?: string } = { diff --git a/apps/web/src/tests/openrouterApi.timeout.test.ts b/apps/web/src/tests/openrouterApi.timeout.test.ts index a436e70d74..3d6758d5cf 100644 --- a/apps/web/src/tests/openrouterApi.timeout.test.ts +++ b/apps/web/src/tests/openrouterApi.timeout.test.ts @@ -54,6 +54,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi, + reasoningEffort: null, search: '?beta=true', method: 'POST', body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, @@ -76,6 +77,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, @@ -89,62 +91,146 @@ describe('upstreamRequest timeout', () => { expect(headers.has('authorization')).toBe(false); }); - it('reports a client disconnect instead of an upstream disconnect when the caller aborts', async () => { - const controller = new AbortController(); - controller.abort(); + it.each(['xhigh', 'max'])( + 'reports a client disconnect without advice for %s effort when the caller aborts', + async reasoningEffort => { + const controller = new AbortController(); + controller.abort(); - const result = await upstreamRequest({ - chatApi: 'chat_completions', - search: '', - method: 'POST', - body: { - model: 'test-model', - messages: [{ role: 'user', content: 'test' }], - }, - extraHeaders: {}, - provider: OPENROUTER, - signal: controller.signal, - }); + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { + model: 'test-model', + messages: [{ role: 'user', content: 'test' }], + }, + extraHeaders: {}, + provider: OPENROUTER, + signal: controller.signal, + reasoningEffort, + }); - expect(result.type).toBe('error'); - expect(mockCaptureException).not.toHaveBeenCalled(); - if (result.type !== 'error') throw new Error('expected an error result'); - expect(result.response.status).toBe(499); - await expect(result.response.json()).resolves.toEqual({ - error: - 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', - error_type: 'client_disconnect', - message: - 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', - }); - }); + expect(result.type).toBe('error'); + expect(mockCaptureException).not.toHaveBeenCalled(); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(499); + await expect(result.response.json()).resolves.toEqual({ + error: + 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', + error_type: 'client_disconnect', + message: + 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', + }); + } + ); - it('reports a gateway timeout message when the upstream sends no response headers', async () => { - const timeoutError = new DOMException( - 'The operation was aborted due to timeout', - 'TimeoutError' - ); - global.fetch = jest.fn().mockRejectedValue(timeoutError); + it.each(['high', 'medium', 'low', 'minimal', 'none', null])( + 'reports a gateway timeout without advice for %s effort', + async reasoningEffort => { + const timeoutError = new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError' + ); + global.fetch = jest.fn().mockRejectedValue(timeoutError); - const result = await upstreamRequest({ - chatApi: 'chat_completions', - search: '', - method: 'POST', - body: { - model: 'test-model', - messages: [{ role: 'user', content: 'test' }], + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { + model: 'test-model', + messages: [{ role: 'user', content: 'test' }], + }, + extraHeaders: {}, + provider: OPENROUTER, + reasoningEffort, + }); + + expect(result.type).toBe('error'); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(503); + await expect(result.response.json()).resolves.toEqual({ + error: 'The upstream provider did not send response headers before the gateway timeout.', + error_type: 'upstream_disconnect', + message: 'The upstream provider did not send response headers before the gateway timeout.', + }); + } + ); + + describe.each(['xhigh', 'max'])('with %s reasoning effort', reasoningEffort => { + test.each([ + { family: 'request_timeout', error: new DOMException('Timed out', 'TimeoutError') }, + { + family: 'headers_timeout', + error: new TypeError('fetch failed', { cause: { code: 'UND_ERR_HEADERS_TIMEOUT' } }), }, - extraHeaders: {}, - provider: OPENROUTER, + { + family: 'connect_timeout', + error: new TypeError('fetch failed', { cause: { code: 'UND_ERR_CONNECT_TIMEOUT' } }), + }, + { + family: 'read_timeout', + error: new TypeError('fetch failed', { cause: { code: 'UND_ERR_BODY_TIMEOUT' } }), + }, + { + family: 'read_timeout', + error: new TypeError('fetch failed', { cause: { code: 'ETIMEDOUT' } }), + }, + ])('appends advice for $family before the request id', async ({ error }) => { + global.fetch = jest.fn().mockRejectedValue(error); + + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, + extraHeaders: {}, + provider: OPENROUTER, + reasoningEffort, + vercelRequestId: 'iad1::iad1::request-id', + }); + + expect(result.type).toBe('error'); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(503); + const message = `The upstream provider did not send response headers before the gateway timeout. Try lowering the reasoning effort from "${reasoningEffort}" to "high" or lower to reduce the chance of timeouts. (request id: iad1::iad1::request-id)`; + await expect(result.response.json()).resolves.toEqual({ + error: message, + error_type: 'upstream_disconnect', + message, + vercel_request_id: 'iad1::iad1::request-id', + }); }); - expect(result.type).toBe('error'); - if (result.type !== 'error') throw new Error('expected an error result'); - expect(result.response.status).toBe(503); - await expect(result.response.json()).resolves.toEqual({ - error: 'The upstream provider did not send response headers before the gateway timeout.', - error_type: 'upstream_disconnect', - message: 'The upstream provider did not send response headers before the gateway timeout.', + test.each([ + { + family: 'conn_reset', + error: new TypeError('fetch failed', { cause: { code: 'ECONNRESET' } }), + }, + { family: 'unknown', error: new TypeError('fetch failed') }, + { family: 'abort', error: new DOMException('Aborted', 'AbortError') }, + ])('does not append advice for $family', async ({ error }) => { + global.fetch = jest.fn().mockRejectedValue(error); + + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, + extraHeaders: {}, + provider: OPENROUTER, + reasoningEffort, + }); + + expect(result.type).toBe('error'); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(503); + await expect(result.response.json()).resolves.toEqual({ + error: 'The upstream provider closed the connection before sending a response.', + error_type: 'upstream_disconnect', + message: 'The upstream provider closed the connection before sending a response.', + }); }); }); @@ -156,6 +242,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -184,6 +271,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -219,6 +307,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -250,6 +339,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -286,6 +376,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -317,6 +408,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '?trace=search-secret', method: 'POST', body: { @@ -366,6 +458,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -403,6 +496,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '?trace=search-secret', method: 'POST', body: {