From 0a98fd7a02fc1bcac392d28aa062e042486bf2e9 Mon Sep 17 00:00:00 2001 From: chrarnoldus <12196001+chrarnoldus@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:06:17 +0000 Subject: [PATCH 1/2] fix: suggest lower reasoning effort on gateway timeouts Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com> --- .../providers/openrouter/request-helpers.ts | 16 ++ .../ai-gateway/providers/upstream-attempt.ts | 2 + .../ai-gateway/providers/upstream-request.ts | 14 +- .../ai-gateway/rewriteModelResponse.test.ts | 105 ++++++++++ .../lib/ai-gateway/rewriteModelResponse.ts | 64 ++++-- .../tests/openrouter-request-helpers.test.ts | 86 +++++++++ .../src/tests/openrouterApi.timeout.test.ts | 182 +++++++++++++----- 7 files changed, 404 insertions(+), 65 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts index dbbfe63253..856c340f2c 100644 --- a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts +++ b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts @@ -19,6 +19,22 @@ export function getMaxTokens(request: GatewayRequest) { return request.body.max_completion_tokens ?? request.body.max_tokens ?? null; } +export function getReasoningEffort(request: GatewayRequest) { + if (request.kind === 'messages') { + return request.body.output_config?.effort ?? null; + } + if (request.kind === 'responses') { + return request.body.reasoning?.effort ?? null; + } + return request.body.reasoning?.effort ?? request.body.reasoning_effort ?? null; +} + +export function getReasoningEffortTimeoutSuggestion(effort: string | null | undefined): string { + return effort === 'xhigh' || effort === 'max' + ? ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts.` + : ''; +} + export function hasMiddleOutTransform(request: GatewayRequest) { return ( (request.kind === 'chat_completions' && request.body.transforms?.includes('middle-out')) || diff --git a/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts b/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts index cf84d5ad03..f1bea7edb3 100644 --- a/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts +++ b/apps/web/src/lib/ai-gateway/providers/upstream-attempt.ts @@ -8,6 +8,7 @@ import { applyProviderSpecificLogic } from '@/lib/ai-gateway/providers/apply-pro import type { GetProviderProviderResult } from '@/lib/ai-gateway/providers/get-provider'; import { isValidOpenRouterModelId } from '@/lib/ai-gateway/providers/gateway-models-cache'; import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; +import { getReasoningEffort } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import { upstreamRequest } from '@/lib/ai-gateway/providers/upstream-request'; import type { FraudDetectionHeaders } from '@/lib/utils'; @@ -92,6 +93,7 @@ export async function sendUpstreamAttempt({ provider: providerContext.provider, signal, vercelRequestId, + reasoningEffort: getReasoningEffort(request), }); if (result.type === 'error') return result; diff --git a/apps/web/src/lib/ai-gateway/providers/upstream-request.ts b/apps/web/src/lib/ai-gateway/providers/upstream-request.ts index bc362f573d..8f1f440238 100644 --- a/apps/web/src/lib/ai-gateway/providers/upstream-request.ts +++ b/apps/web/src/lib/ai-gateway/providers/upstream-request.ts @@ -9,6 +9,7 @@ import type { GatewayMessagesRequest, } from '@/lib/ai-gateway/providers/openrouter/types'; import { ATTRIBUTION_HEADERS } from '@/lib/ai-gateway/providers/openrouter/attribution-headers'; +import { getReasoningEffortTimeoutSuggestion } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import type { GatewayChatApiKind, Provider } from '@/lib/ai-gateway/providers/types'; import { after, NextResponse } from 'next/server'; import { ProxyErrorType } from '@/lib/proxy-error-types'; @@ -172,14 +173,15 @@ function clientDisconnectResponse(vercelRequestId: string | null | undefined) { function upstreamFetchFailureResponse( failureFamily: UpstreamFetchFailureFamily, - vercelRequestId: string | null | undefined + vercelRequestId: string | null | undefined, + reasoningEffort: string | null | undefined ) { const error = withRequestId( failureFamily === 'request_timeout' || failureFamily === 'headers_timeout' || failureFamily === 'connect_timeout' || failureFamily === 'read_timeout' - ? 'The upstream provider did not send response headers before the gateway timeout.' + ? `The upstream provider did not send response headers before the gateway timeout.${getReasoningEffortTimeoutSuggestion(reasoningEffort)}` : 'The upstream provider closed the connection before sending a response.', vercelRequestId ); @@ -203,6 +205,7 @@ export async function upstreamRequest({ provider, signal, vercelRequestId, + reasoningEffort, }: { chatApi: GatewayChatApiKind; search: string; @@ -213,6 +216,7 @@ export async function upstreamRequest({ signal?: AbortSignal; /** Incoming `x-vercel-id`, used to correlate failures with the platform logs. */ vercelRequestId?: string | null; + reasoningEffort?: string | null; }): Promise<{ type: 'success'; response: Response } | { type: 'error'; response: NextResponse }> { const headers = new Headers(); for (const [key, value] of Object.entries(ATTRIBUTION_HEADERS)) { @@ -309,7 +313,11 @@ export async function upstreamRequest({ type: 'error', response: causedByClientDisconnect ? clientDisconnectResponse(vercelRequestId) - : upstreamFetchFailureResponse(failureFamily ?? 'unknown', vercelRequestId), + : upstreamFetchFailureResponse( + failureFamily ?? 'unknown', + vercelRequestId, + reasoningEffort + ), }; } } diff --git a/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts b/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts index e6e22d49b0..b8f871aa2b 100644 --- a/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts +++ b/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts @@ -14,6 +14,7 @@ import { QWEN37_PLUS_MODEL_ID } from '@/lib/ai-gateway/custom-pricing'; import { KILO_ORGANIZATION_ID } from '@/lib/organizations/constants'; import { logExceptInTest } from '@/lib/utils.server'; import { ReasoningDetailsTransform } from '@/lib/ai-gateway/providers/types'; +import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; jest.mock('next/server', () => ({ ...(jest.requireActual('next/server') as Record), @@ -151,6 +152,25 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { }); }); + test.each(['high', 'medium', 'low', 'none', null, undefined])( + 'omits effort advice for effort %s', + async reasoningEffort => { + const result = await rewrite({ + response: failingResponse('application/json', 'TimeoutError'), + removeCost: true, + capture: null, + vercelRequestId: null, + reasoningEffort, + }); + + expect(await result.json()).toEqual({ + error: 'The upstream provider timed out while sending the response.', + error_type: 'timeout', + message: 'The upstream provider timed out while sending the response.', + }); + } + ); + test('includes the vercel request id only in the JSON read error message', async () => { const result = await rewrite({ response: failingResponse('application/json', 'ResponseAborted'), @@ -204,6 +224,91 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { }); }); +const timeoutEffortRequests = [ + { + name: 'Chat Completions nested effort', + request: { + kind: 'chat_completions', + body: { model: 'test-model', messages: [], reasoning: { effort: 'xhigh' } }, + }, + effort: 'xhigh', + }, + { + name: 'Chat Completions top-level effort', + request: { + kind: 'chat_completions', + body: { model: 'test-model', messages: [], reasoning_effort: 'max' }, + }, + effort: 'max', + }, + { + name: 'Responses', + request: { + kind: 'responses', + body: { model: 'test-model', input: 'Hello', reasoning: { effort: 'xhigh' } }, + }, + effort: 'xhigh', + }, + { + name: 'Messages', + request: { + kind: 'messages', + body: { + model: 'test-model', + messages: [], + max_tokens: 1024, + output_config: { effort: 'max' }, + }, + }, + effort: 'max', + }, +] satisfies { name: string; request: GatewayRequest; effort: string }[]; + +describe.each(timeoutEffortRequests)('$name timeout effort guidance', ({ request, effort }) => { + test.each([ + ['application/json', 'TimeoutError', 'timeout'], + ['application/json', 'ResponseAborted', 'upstream_disconnect'], + ['text/event-stream', 'TimeoutError', 'timeout'], + ['text/event-stream', 'ResponseAborted', 'upstream_disconnect'], + ])('suggests lowering effort for %s %s', async (contentType, errorName, errorType) => { + const result = await rewriteModelResponse({ + response: failingResponse(contentType, errorName), + model: request.body.model ?? 'test-model', + providerId: 'openrouter', + kind: request.kind, + logging: { + user: null, + organization_id: null, + session_id: null, + vercel_request_id: 'fra1::request-id', + request, + }, + responseTransforms: null, + }); + const expectedMessage = + (errorName === 'TimeoutError' + ? 'The upstream provider timed out while sending the response.' + : 'The upstream response was interrupted while streaming. The provider may have disconnected or the request may have timed out.') + + ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts. (request id: fra1::request-id)`; + + if (contentType === 'application/json') { + expect(result.status).toBe(503); + expect(await result.json()).toEqual({ + error: expectedMessage, + error_type: errorType, + message: expectedMessage, + }); + } else { + expect(result.status).toBe(200); + expect(dataObjects(await readOutputStream(result))).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ message: expectedMessage }), + }), + ]); + } + }); +}); + describe('rewriteModelResponse_ChatCompletions', () => { describe('JSON responses', () => { test('strips upstream cost fields', async () => { diff --git a/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts b/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts index 765159a2aa..98c515375f 100644 --- a/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts +++ b/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts @@ -3,6 +3,10 @@ import { isKiloExclusiveFreeModel } from '@/lib/ai-gateway/models'; import { getCustomPricing } from '@/lib/ai-gateway/custom-pricing'; import { detectToolCallArgumentErrors } from '@/lib/ai-gateway/api-request-log-errors'; import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; +import { + getReasoningEffort, + getReasoningEffortTimeoutSuggestion, +} from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import type { ProviderId, ProviderResponseTransforms } from '@/lib/ai-gateway/providers/types'; import { getOutputHeaders } from '@/lib/ai-gateway/llm-proxy-helpers'; import type { ChatCompletionChunk, OpenRouterUsage } from '@/lib/ai-gateway/processUsage.types'; @@ -49,6 +53,7 @@ export type RewriteResponseParams = { removeCost: boolean; capture: RequestLogCapture | null; vercelRequestId: string | null; + reasoningEffort?: string | null; }; export type RewriteChatCompletionsResponseParams = RewriteResponseParams & { @@ -247,7 +252,8 @@ function logTerminalStreamEvent( function getResponseReadError( error: unknown, - vercelRequestId: string | null | undefined + vercelRequestId: string | null | undefined, + reasoningEffort: string | null | undefined ): ResponseReadError | null { if (typeof error !== 'object' || error === null || !('name' in error)) { return null; @@ -257,7 +263,8 @@ function getResponseReadError( return { errorType: 'upstream_disconnect', message: withRequestId( - 'The upstream response was interrupted while streaming. The provider may have disconnected or the request may have timed out.', + 'The upstream response was interrupted while streaming. The provider may have disconnected or the request may have timed out.' + + getReasoningEffortTimeoutSuggestion(reasoningEffort), vercelRequestId ), }; @@ -267,7 +274,8 @@ function getResponseReadError( return { errorType: 'timeout', message: withRequestId( - 'The upstream provider timed out while sending the response.', + 'The upstream provider timed out while sending the response.' + + getReasoningEffortTimeoutSuggestion(reasoningEffort), vercelRequestId ), }; @@ -280,12 +288,13 @@ async function readResponseText( response: Response, headers: Headers, vercelRequestId: string | null | undefined, - capture: RequestLogCapture | null + capture: RequestLogCapture | null, + reasoningEffort: string | null | undefined ): Promise<{ text: string } | { error: unknown; errorResponse: NextResponse }> { try { return { text: await response.text() }; } catch (error) { - const responseReadError = getResponseReadError(error, vercelRequestId); + const responseReadError = getResponseReadError(error, vercelRequestId, reasoningEffort); if (!responseReadError) { // Settle the capture so the after() callback awaiting it does not hang // and the request is still logged (without a response body). @@ -321,7 +330,8 @@ async function rewriteSseStream( onFinally: () => void, vercelRequestId: string | null | undefined, capture: RequestLogCapture | null, - capturedChunks: string[] | null + capturedChunks: string[] | null, + reasoningEffort: string | null | undefined ) { const decoder = new TextDecoder(); const settleReadError = (error: unknown) => @@ -368,7 +378,7 @@ async function rewriteSseStream( } } } catch (error) { - const responseReadError = getResponseReadError(error, vercelRequestId); + const responseReadError = getResponseReadError(error, vercelRequestId, reasoningEffort); if (!responseReadError) { settleReadError(error); throw error; @@ -404,6 +414,7 @@ export async function rewriteModelResponse_ChatCompletions({ response, removeCost, capture, + reasoningEffort, vercelRequestId, responseTransforms, }: RewriteChatCompletionsResponseParams) { @@ -412,7 +423,13 @@ export async function rewriteModelResponse_ChatCompletions({ if (headers.get('content-type')?.includes('application/json')) { // Read the body text once to avoid "Response body object should not be // disturbed or locked" errors that occur when `.clone().json()` fails. - const textResult = await readResponseText(response, headers, vercelRequestId, capture); + const textResult = await readResponseText( + response, + headers, + vercelRequestId, + capture, + reasoningEffort + ); if ('errorResponse' in textResult) { capture?.setReadError(textResult.error); return textResult.errorResponse; @@ -539,7 +556,8 @@ export async function rewriteModelResponse_ChatCompletions({ progress.stop, vercelRequestId, capture, - capturedChunks + capturedChunks, + reasoningEffort ); }, cancel() { @@ -591,12 +609,19 @@ export async function rewriteModelResponse_Messages({ response, removeCost, capture, + reasoningEffort, vercelRequestId, }: RewriteResponseParams) { const headers = getOutputHeaders(response); if (headers.get('content-type')?.includes('application/json')) { - const textResult = await readResponseText(response, headers, vercelRequestId, capture); + const textResult = await readResponseText( + response, + headers, + vercelRequestId, + capture, + reasoningEffort + ); if ('errorResponse' in textResult) { capture?.setReadError(textResult.error); return textResult.errorResponse; @@ -719,7 +744,8 @@ export async function rewriteModelResponse_Messages({ progress.stop, vercelRequestId, capture, - capturedChunks + capturedChunks, + reasoningEffort ); }, cancel() { @@ -747,12 +773,19 @@ export async function rewriteModelResponse_Responses({ response, removeCost, capture, + reasoningEffort, vercelRequestId, }: RewriteResponseParams) { const headers = getOutputHeaders(response); if (headers.get('content-type')?.includes('application/json')) { - const textResult = await readResponseText(response, headers, vercelRequestId, capture); + const textResult = await readResponseText( + response, + headers, + vercelRequestId, + capture, + reasoningEffort + ); if ('errorResponse' in textResult) { capture?.setReadError(textResult.error); return textResult.errorResponse; @@ -870,7 +903,8 @@ export async function rewriteModelResponse_Responses({ progress.stop, vercelRequestId, capture, - capturedChunks + capturedChunks, + reasoningEffort ); }, cancel() { @@ -905,11 +939,13 @@ export async function rewriteModelResponse({ console.debug('[rewriteModelResponse] rewriting response for %s', model); const { vercel_request_id: vercelRequestId } = logging; + const reasoningEffort = getReasoningEffort(logging.request); if (kind === 'chat_completions') { return rewriteModelResponse_ChatCompletions({ response, removeCost: requiresCostRemoval, capture, + reasoningEffort, vercelRequestId, responseTransforms, }); @@ -919,6 +955,7 @@ export async function rewriteModelResponse({ response, removeCost: requiresCostRemoval, capture, + reasoningEffort, vercelRequestId, }); } @@ -927,6 +964,7 @@ export async function rewriteModelResponse({ response, removeCost: requiresCostRemoval, capture, + reasoningEffort, vercelRequestId, }); } diff --git a/apps/web/src/tests/openrouter-request-helpers.test.ts b/apps/web/src/tests/openrouter-request-helpers.test.ts index 68f02fbc86..66e1d3f9f3 100644 --- a/apps/web/src/tests/openrouter-request-helpers.test.ts +++ b/apps/web/src/tests/openrouter-request-helpers.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from '@jest/globals'; import { addCacheBreakpoints, + getReasoningEffort, + getReasoningEffortTimeoutSuggestion, removeChatCompletionsToolNames, } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; import type { @@ -9,6 +11,90 @@ import type { } from '@/lib/ai-gateway/providers/openrouter/types'; import type OpenAI from 'openai'; +describe('getReasoningEffort', () => { + test.each([ + { options: { output_config: { effort: 'max' } }, expected: 'max' }, + { options: { output_config: { effort: 'high' } }, expected: 'high' }, + { options: { output_config: {} }, expected: null }, + { options: {}, expected: null }, + ] as const)('reads messages effort from $options as $expected', ({ options, expected }) => { + expect( + getReasoningEffort({ + kind: 'messages', + body: { model: 'test-model', messages: [], max_tokens: 1024, ...options }, + }) + ).toBe(expected); + }); + + test.each([ + { options: { reasoning: { effort: 'xhigh' } }, expected: 'xhigh' }, + { options: { reasoning: { effort: 'high' } }, expected: 'high' }, + { options: { reasoning: { effort: null } }, expected: null }, + { options: { reasoning: {} }, expected: null }, + { options: { reasoning: null }, expected: null }, + { options: {}, expected: null }, + ] as const)('reads responses effort from $options as $expected', ({ options, expected }) => { + expect( + getReasoningEffort({ + kind: 'responses', + body: { model: 'test-model', input: 'test', ...options }, + }) + ).toBe(expected); + }); + + test.each([ + { options: { reasoning: { effort: 'xhigh' } }, expected: 'xhigh' }, + { options: { reasoning_effort: 'xhigh' }, expected: 'xhigh' }, + { + options: { reasoning: { effort: 'high' }, reasoning_effort: 'xhigh' }, + expected: 'high', + }, + { + options: { reasoning: { effort: 'none' }, reasoning_effort: 'xhigh' }, + expected: 'none', + }, + { options: { reasoning: { effort: null }, reasoning_effort: 'xhigh' }, expected: 'xhigh' }, + { options: { reasoning: {}, reasoning_effort: 'high' }, expected: 'high' }, + { options: { reasoning: { effort: null }, reasoning_effort: null }, expected: null }, + { options: { reasoning: {} }, expected: null }, + { options: {}, expected: null }, + ] as const)( + 'reads chat completions effort from $options as $expected', + ({ options, expected }) => { + expect( + getReasoningEffort({ + kind: 'chat_completions', + body: { model: 'test-model', messages: [], ...options }, + }) + ).toBe(expected); + } + ); +}); + +describe('getReasoningEffortTimeoutSuggestion', () => { + test.each(['xhigh', 'max'])('suggests lowering %s effort', effort => { + expect(getReasoningEffortTimeoutSuggestion(effort)).toBe( + ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts.` + ); + }); + + test.each([ + 'high', + 'medium', + 'low', + 'minimal', + 'none', + 'XHIGH', + 'MAX', + 'max ', + '', + null, + undefined, + ])('does not suggest lowering %s effort', effort => { + expect(getReasoningEffortTimeoutSuggestion(effort)).toBe(''); + }); +}); + describe('removeChatCompletionsToolNames', () => { test('removes names from tool messages only', () => { const toolMessage: OpenAI.ChatCompletionToolMessageParam & { name?: string } = { diff --git a/apps/web/src/tests/openrouterApi.timeout.test.ts b/apps/web/src/tests/openrouterApi.timeout.test.ts index a436e70d74..cd57e5b7e6 100644 --- a/apps/web/src/tests/openrouterApi.timeout.test.ts +++ b/apps/web/src/tests/openrouterApi.timeout.test.ts @@ -89,62 +89,146 @@ describe('upstreamRequest timeout', () => { expect(headers.has('authorization')).toBe(false); }); - it('reports a client disconnect instead of an upstream disconnect when the caller aborts', async () => { - const controller = new AbortController(); - controller.abort(); + it.each(['xhigh', 'max'])( + 'reports a client disconnect without advice for %s effort when the caller aborts', + async reasoningEffort => { + const controller = new AbortController(); + controller.abort(); - const result = await upstreamRequest({ - chatApi: 'chat_completions', - search: '', - method: 'POST', - body: { - model: 'test-model', - messages: [{ role: 'user', content: 'test' }], - }, - extraHeaders: {}, - provider: OPENROUTER, - signal: controller.signal, - }); + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { + model: 'test-model', + messages: [{ role: 'user', content: 'test' }], + }, + extraHeaders: {}, + provider: OPENROUTER, + signal: controller.signal, + reasoningEffort, + }); - expect(result.type).toBe('error'); - expect(mockCaptureException).not.toHaveBeenCalled(); - if (result.type !== 'error') throw new Error('expected an error result'); - expect(result.response.status).toBe(499); - await expect(result.response.json()).resolves.toEqual({ - error: - 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', - error_type: 'client_disconnect', - message: - 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', - }); - }); + expect(result.type).toBe('error'); + expect(mockCaptureException).not.toHaveBeenCalled(); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(499); + await expect(result.response.json()).resolves.toEqual({ + error: + 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', + error_type: 'client_disconnect', + message: + 'The client disconnected before the upstream provider responded, so the request was cancelled. The upstream provider did not fail.', + }); + } + ); - it('reports a gateway timeout message when the upstream sends no response headers', async () => { - const timeoutError = new DOMException( - 'The operation was aborted due to timeout', - 'TimeoutError' - ); - global.fetch = jest.fn().mockRejectedValue(timeoutError); + it.each(['high', 'medium', 'low', 'minimal', 'none', null, undefined])( + 'reports a gateway timeout without advice for %s effort', + async reasoningEffort => { + const timeoutError = new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError' + ); + global.fetch = jest.fn().mockRejectedValue(timeoutError); - const result = await upstreamRequest({ - chatApi: 'chat_completions', - search: '', - method: 'POST', - body: { - model: 'test-model', - messages: [{ role: 'user', content: 'test' }], + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { + model: 'test-model', + messages: [{ role: 'user', content: 'test' }], + }, + extraHeaders: {}, + provider: OPENROUTER, + reasoningEffort, + }); + + expect(result.type).toBe('error'); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(503); + await expect(result.response.json()).resolves.toEqual({ + error: 'The upstream provider did not send response headers before the gateway timeout.', + error_type: 'upstream_disconnect', + message: 'The upstream provider did not send response headers before the gateway timeout.', + }); + } + ); + + describe.each(['xhigh', 'max'])('with %s reasoning effort', reasoningEffort => { + test.each([ + { family: 'request_timeout', error: new DOMException('Timed out', 'TimeoutError') }, + { + family: 'headers_timeout', + error: new TypeError('fetch failed', { cause: { code: 'UND_ERR_HEADERS_TIMEOUT' } }), }, - extraHeaders: {}, - provider: OPENROUTER, + { + family: 'connect_timeout', + error: new TypeError('fetch failed', { cause: { code: 'UND_ERR_CONNECT_TIMEOUT' } }), + }, + { + family: 'read_timeout', + error: new TypeError('fetch failed', { cause: { code: 'UND_ERR_BODY_TIMEOUT' } }), + }, + { + family: 'read_timeout', + error: new TypeError('fetch failed', { cause: { code: 'ETIMEDOUT' } }), + }, + ])('appends advice for $family before the request id', async ({ error }) => { + global.fetch = jest.fn().mockRejectedValue(error); + + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, + extraHeaders: {}, + provider: OPENROUTER, + reasoningEffort, + vercelRequestId: 'iad1::iad1::request-id', + }); + + expect(result.type).toBe('error'); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(503); + const message = `The upstream provider did not send response headers before the gateway timeout. Try lowering the reasoning effort from "${reasoningEffort}" to "high" or lower to reduce the chance of timeouts. (request id: iad1::iad1::request-id)`; + await expect(result.response.json()).resolves.toEqual({ + error: message, + error_type: 'upstream_disconnect', + message, + vercel_request_id: 'iad1::iad1::request-id', + }); }); - expect(result.type).toBe('error'); - if (result.type !== 'error') throw new Error('expected an error result'); - expect(result.response.status).toBe(503); - await expect(result.response.json()).resolves.toEqual({ - error: 'The upstream provider did not send response headers before the gateway timeout.', - error_type: 'upstream_disconnect', - message: 'The upstream provider did not send response headers before the gateway timeout.', + test.each([ + { + family: 'conn_reset', + error: new TypeError('fetch failed', { cause: { code: 'ECONNRESET' } }), + }, + { family: 'unknown', error: new TypeError('fetch failed') }, + { family: 'abort', error: new DOMException('Aborted', 'AbortError') }, + ])('does not append advice for $family', async ({ error }) => { + global.fetch = jest.fn().mockRejectedValue(error); + + const result = await upstreamRequest({ + chatApi: 'chat_completions', + search: '', + method: 'POST', + body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, + extraHeaders: {}, + provider: OPENROUTER, + reasoningEffort, + }); + + expect(result.type).toBe('error'); + if (result.type !== 'error') throw new Error('expected an error result'); + expect(result.response.status).toBe(503); + await expect(result.response.json()).resolves.toEqual({ + error: 'The upstream provider closed the connection before sending a response.', + error_type: 'upstream_disconnect', + message: 'The upstream provider closed the connection before sending a response.', + }); }); }); From ae265be51160d693b9e62e10299dd9950908c625 Mon Sep 17 00:00:00 2001 From: chrarnoldus <12196001+chrarnoldus@users.noreply.github.com> Date: Mon, 31 Aug 2026 14:47:11 +0000 Subject: [PATCH 2/2] refactor: require nullable reasoning effort parameters Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com> --- .../providers/openrouter/request-helpers.ts | 2 +- .../ai-gateway/providers/upstream-request.ts | 4 +- .../ai-gateway/rewriteModelResponse.test.ts | 43 ++++++++++++++++++- .../lib/ai-gateway/rewriteModelResponse.ts | 8 ++-- .../tests/openrouter-request-helpers.test.ts | 21 +++------ .../src/tests/openrouterApi.timeout.test.ts | 12 +++++- 6 files changed, 66 insertions(+), 24 deletions(-) diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts index 856c340f2c..60a0212a24 100644 --- a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts +++ b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts @@ -29,7 +29,7 @@ export function getReasoningEffort(request: GatewayRequest) { return request.body.reasoning?.effort ?? request.body.reasoning_effort ?? null; } -export function getReasoningEffortTimeoutSuggestion(effort: string | null | undefined): string { +export function getReasoningEffortTimeoutSuggestion(effort: string | null): string { return effort === 'xhigh' || effort === 'max' ? ` Try lowering the reasoning effort from "${effort}" to "high" or lower to reduce the chance of timeouts.` : ''; diff --git a/apps/web/src/lib/ai-gateway/providers/upstream-request.ts b/apps/web/src/lib/ai-gateway/providers/upstream-request.ts index 8f1f440238..b26cb10032 100644 --- a/apps/web/src/lib/ai-gateway/providers/upstream-request.ts +++ b/apps/web/src/lib/ai-gateway/providers/upstream-request.ts @@ -174,7 +174,7 @@ function clientDisconnectResponse(vercelRequestId: string | null | undefined) { function upstreamFetchFailureResponse( failureFamily: UpstreamFetchFailureFamily, vercelRequestId: string | null | undefined, - reasoningEffort: string | null | undefined + reasoningEffort: string | null ) { const error = withRequestId( failureFamily === 'request_timeout' || @@ -216,7 +216,7 @@ export async function upstreamRequest({ signal?: AbortSignal; /** Incoming `x-vercel-id`, used to correlate failures with the platform logs. */ vercelRequestId?: string | null; - reasoningEffort?: string | null; + reasoningEffort: string | null; }): Promise<{ type: 'success'; response: Response } | { type: 'error'; response: NextResponse }> { const headers = new Headers(); for (const [key, value] of Object.entries(ATTRIBUTION_HEADERS)) { diff --git a/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts b/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts index b8f871aa2b..b42f2438f8 100644 --- a/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts +++ b/apps/web/src/lib/ai-gateway/rewriteModelResponse.test.ts @@ -142,6 +142,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(503); @@ -152,7 +153,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { }); }); - test.each(['high', 'medium', 'low', 'none', null, undefined])( + test.each(['high', 'medium', 'low', 'none', null])( 'omits effort advice for effort %s', async reasoningEffort => { const result = await rewrite({ @@ -177,6 +178,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: 'iad1::iad1::request-id', + reasoningEffort: null, }); expect(result.status).toBe(503); @@ -195,6 +197,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: 'iad1::iad1::request-id', + reasoningEffort: null, }); const events = dataObjects(await readOutputStream(result)) as { error: { message: string }; @@ -213,6 +216,7 @@ describe.each(rewriters)('%s response read errors', (_name, rewrite) => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const events = dataObjects(await readOutputStream(result)) as { error: { message: string }; @@ -330,6 +334,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const json = await result.json(); @@ -359,6 +364,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const json = await result.json(); @@ -378,6 +384,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); @@ -403,6 +410,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.ReasoningContent, }); const json = await result.json(); @@ -438,6 +446,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.ReasoningContent, }); const json = await result.json(); @@ -465,6 +474,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const json = await result.json(); @@ -500,6 +510,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const reader = result.body?.getReader(); @@ -531,6 +542,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -553,6 +565,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -578,6 +591,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.GeminiThought, }); const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ @@ -627,6 +641,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.GeminiThought, }); const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ @@ -657,6 +672,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: ReasoningDetailsTransform.ReasoningContent, }); const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ @@ -688,6 +704,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -709,6 +726,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture, vercelRequestId: 'iad1::terminal-request', + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -739,6 +757,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -769,6 +788,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const sse = await readOutputStream(result); @@ -787,6 +807,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); @@ -809,6 +830,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -851,6 +873,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const json = await result.json(); @@ -872,6 +895,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(500); @@ -890,6 +914,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); const events = dataObjects(sse) as Array<{ @@ -924,6 +949,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -942,6 +968,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -973,6 +1000,7 @@ describe('rewriteModelResponse_Messages', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -998,6 +1026,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -1040,6 +1069,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const json = await result.json(); @@ -1061,6 +1091,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture: null, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); const [event] = dataObjects(sse) as Array<{ @@ -1097,6 +1128,7 @@ describe('rewriteModelResponse_Responses', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -1155,6 +1187,7 @@ describe.each([ removeCost: true, capture, vercelRequestId: 'iad1::error-request', + reasoningEffort: null, }); const sse = await readOutputStream(result); @@ -1420,6 +1453,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(200); @@ -1438,6 +1472,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1456,6 +1491,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1475,6 +1511,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1494,6 +1531,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); await readOutputStream(result); @@ -1513,6 +1551,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); expect(result.status).toBe(503); @@ -1532,6 +1571,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, responseTransforms: null, }); const reader = result.body?.getReader(); @@ -1553,6 +1593,7 @@ describe('request log capture', () => { removeCost: true, capture, vercelRequestId: null, + reasoningEffort: null, }); const reader = result.body?.getReader(); await reader?.read(); diff --git a/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts b/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts index 98c515375f..645b91dd70 100644 --- a/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts +++ b/apps/web/src/lib/ai-gateway/rewriteModelResponse.ts @@ -53,7 +53,7 @@ export type RewriteResponseParams = { removeCost: boolean; capture: RequestLogCapture | null; vercelRequestId: string | null; - reasoningEffort?: string | null; + reasoningEffort: string | null; }; export type RewriteChatCompletionsResponseParams = RewriteResponseParams & { @@ -253,7 +253,7 @@ function logTerminalStreamEvent( function getResponseReadError( error: unknown, vercelRequestId: string | null | undefined, - reasoningEffort: string | null | undefined + reasoningEffort: string | null ): ResponseReadError | null { if (typeof error !== 'object' || error === null || !('name' in error)) { return null; @@ -289,7 +289,7 @@ async function readResponseText( headers: Headers, vercelRequestId: string | null | undefined, capture: RequestLogCapture | null, - reasoningEffort: string | null | undefined + reasoningEffort: string | null ): Promise<{ text: string } | { error: unknown; errorResponse: NextResponse }> { try { return { text: await response.text() }; @@ -331,7 +331,7 @@ async function rewriteSseStream( vercelRequestId: string | null | undefined, capture: RequestLogCapture | null, capturedChunks: string[] | null, - reasoningEffort: string | null | undefined + reasoningEffort: string | null ) { const decoder = new TextDecoder(); const settleReadError = (error: unknown) => diff --git a/apps/web/src/tests/openrouter-request-helpers.test.ts b/apps/web/src/tests/openrouter-request-helpers.test.ts index 66e1d3f9f3..7b041b6658 100644 --- a/apps/web/src/tests/openrouter-request-helpers.test.ts +++ b/apps/web/src/tests/openrouter-request-helpers.test.ts @@ -78,21 +78,12 @@ describe('getReasoningEffortTimeoutSuggestion', () => { ); }); - test.each([ - 'high', - 'medium', - 'low', - 'minimal', - 'none', - 'XHIGH', - 'MAX', - 'max ', - '', - null, - undefined, - ])('does not suggest lowering %s effort', effort => { - expect(getReasoningEffortTimeoutSuggestion(effort)).toBe(''); - }); + test.each(['high', 'medium', 'low', 'minimal', 'none', 'XHIGH', 'MAX', 'max ', '', null])( + 'does not suggest lowering %s effort', + effort => { + expect(getReasoningEffortTimeoutSuggestion(effort)).toBe(''); + } + ); }); describe('removeChatCompletionsToolNames', () => { diff --git a/apps/web/src/tests/openrouterApi.timeout.test.ts b/apps/web/src/tests/openrouterApi.timeout.test.ts index cd57e5b7e6..3d6758d5cf 100644 --- a/apps/web/src/tests/openrouterApi.timeout.test.ts +++ b/apps/web/src/tests/openrouterApi.timeout.test.ts @@ -54,6 +54,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi, + reasoningEffort: null, search: '?beta=true', method: 'POST', body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, @@ -76,6 +77,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { model: 'test-model', messages: [{ role: 'user', content: 'test' }] }, @@ -123,7 +125,7 @@ describe('upstreamRequest timeout', () => { } ); - it.each(['high', 'medium', 'low', 'minimal', 'none', null, undefined])( + it.each(['high', 'medium', 'low', 'minimal', 'none', null])( 'reports a gateway timeout without advice for %s effort', async reasoningEffort => { const timeoutError = new DOMException( @@ -240,6 +242,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -268,6 +271,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -303,6 +307,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -334,6 +339,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -370,6 +376,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -401,6 +408,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '?trace=search-secret', method: 'POST', body: { @@ -450,6 +458,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '', method: 'POST', body: { @@ -487,6 +496,7 @@ describe('upstreamRequest timeout', () => { const result = await upstreamRequest({ chatApi: 'chat_completions', + reasoningEffort: null, search: '?trace=search-secret', method: 'POST', body: {