From 3dbc998be977f24a1a82008d7b0de4957a90fde2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Sat, 29 Aug 2026 14:46:30 +0200 Subject: [PATCH] feat(integrations): bound repository transport for harness reads --- .../core/repository-read-limits.test.ts | 77 +++++++++ .../core/repository-read-limits.ts | 102 +++++++++++ .../bitbucket/token-service-client.test.ts | 124 +++++++++++++- .../bitbucket/token-service-client.ts | 85 ++++++---- .../github/adapter.repositories.test.ts | 130 ++++++++++++++ .../integrations/platforms/github/adapter.ts | 92 +++++++--- .../platforms/gitlab/adapter.test.ts | 115 +++++++++++++ .../integrations/platforms/gitlab/adapter.ts | 160 ++++++++++++------ 8 files changed, 771 insertions(+), 114 deletions(-) create mode 100644 apps/web/src/lib/integrations/core/repository-read-limits.test.ts create mode 100644 apps/web/src/lib/integrations/core/repository-read-limits.ts create mode 100644 apps/web/src/lib/integrations/platforms/github/adapter.repositories.test.ts diff --git a/apps/web/src/lib/integrations/core/repository-read-limits.test.ts b/apps/web/src/lib/integrations/core/repository-read-limits.test.ts new file mode 100644 index 0000000000..932a4665ca --- /dev/null +++ b/apps/web/src/lib/integrations/core/repository-read-limits.test.ts @@ -0,0 +1,77 @@ +import { boundRepositoryResponse, withRepositoryReadDeadline } from './repository-read-limits'; + +const jsonHeaders = { 'content-type': 'application/json' }; + +afterEach(() => jest.useRealTimers()); + +describe('repository response bounds', () => { + it('accepts exactly 1 MiB without changing the decoded data', async () => { + const body = JSON.stringify('x'.repeat(1024 * 1024 - 2)); + const response = await boundRepositoryResponse(new Response(body, { headers: jsonHeaders })); + expect(await response.json()).toHaveLength(1024 * 1024 - 2); + }); + + it.each(['stream', 'advertised', 'invalid length', 'content type', 'invalid bytes'])( + 'rejects and cancels %s before parsing', + async failure => { + let cancelled = false; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + failure === 'invalid bytes' ? new Uint8Array([255]) : new Uint8Array(1024 * 1024 + 1) + ); + if (failure === 'invalid bytes') controller.close(); + }, + cancel() { + cancelled = true; + }, + }); + const headers = { + ...jsonHeaders, + ...(failure === 'advertised' + ? { 'content-length': '1048577' } + : failure === 'invalid length' + ? { 'content-length': 'invalid' } + : {}), + }; + if (failure === 'content type') headers['content-type'] = 'text/html'; + await expect(boundRepositoryResponse(new Response(stream, { headers }))).rejects.toThrow(); + if (failure !== 'invalid bytes') expect(cancelled).toBe(true); + } + ); + + it('cancels a stalled body at the operation deadline', async () => { + jest.useFakeTimers(); + let cancelled = false; + const response = new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { headers: jsonHeaders } + ); + const result = withRepositoryReadDeadline({ bounded: true }, signal => + boundRepositoryResponse(response, signal) + ); + const rejection = expect(result).rejects.toThrow('Repository fetch timed out'); + await jest.advanceTimersByTimeAsync(30_000); + await rejection; + expect(cancelled).toBe(true); + }); + + it('propagates caller cancellation without waiting for the deadline', async () => { + const controller = new AbortController(); + const result = withRepositoryReadDeadline( + { bounded: true, signal: controller.signal }, + async signal => { + await new Promise(resolve => + signal?.addEventListener('abort', () => resolve(), { once: true }) + ); + signal?.throwIfAborted(); + } + ); + controller.abort(new Error('Read cancelled')); + await expect(result).rejects.toThrow('Read cancelled'); + }); +}); diff --git a/apps/web/src/lib/integrations/core/repository-read-limits.ts b/apps/web/src/lib/integrations/core/repository-read-limits.ts new file mode 100644 index 0000000000..35e52f818d --- /dev/null +++ b/apps/web/src/lib/integrations/core/repository-read-limits.ts @@ -0,0 +1,102 @@ +import { z } from 'zod'; + +export const REPOSITORY_READ_LIMITS = { + pages: 2, + repositories: 50, + responseBytes: 1024 * 1024, + timeoutMs: 30_000, +} as const; + +// Omitted options preserve complete legacy reads until those callers retire. +// Bounded results can be incomplete and must not replace a complete shared cache. +export type RepositoryReadOptions = { bounded?: boolean; signal?: AbortSignal }; + +export const repositoryPageSchema = z.custom( + value => Array.isArray(value) && value.length <= REPOSITORY_READ_LIMITS.repositories, + 'Invalid repository page' +); + +export async function withRepositoryReadDeadline( + options: RepositoryReadOptions | undefined, + read: (signal?: AbortSignal) => Promise +): Promise { + if (!options?.bounded) return read(); + const controller = new AbortController(); + const signal = options.signal + ? AbortSignal.any([controller.signal, options.signal]) + : controller.signal; + const timer = setTimeout( + () => controller.abort(new Error('Repository fetch timed out')), + REPOSITORY_READ_LIMITS.timeoutMs + ); + const aborted = Promise.withResolvers(); + const onAbort = () => aborted.reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + try { + signal.throwIfAborted(); + return await Promise.race([read(signal), aborted.promise]); + } finally { + clearTimeout(timer); + signal.removeEventListener('abort', onAbort); + } +} + +export async function boundRepositoryResponse( + response: Response, + signal?: AbortSignal +): Promise { + if (!response.body) return response; + const reader = response.body.getReader(); + const cancel = () => { + void reader.cancel(signal?.reason).catch(() => {}); + }; + signal?.addEventListener('abort', cancel, { once: true }); + try { + signal?.throwIfAborted(); + const length = response.headers.get('content-length'); + if ( + length && + (!/^\d+$/.test(length) || Number(length) > REPOSITORY_READ_LIMITS.responseBytes) + ) { + throw new Error('Repository response exceeded size limit'); + } + if ( + response.ok && + response.headers.get('content-type')?.split(';')[0].trim().toLowerCase() !== + 'application/json' + ) { + throw new Error('Invalid repository response content type'); + } + const bytes = new Uint8Array(REPOSITORY_READ_LIMITS.responseBytes); + let size = 0; + while (true) { + const chunk = await reader.read(); + signal?.throwIfAborted(); + if (chunk.done) break; + if (!(chunk.value instanceof Uint8Array)) throw new Error('Invalid repository response'); + if (size + chunk.value.byteLength > bytes.byteLength) { + throw new Error('Repository response exceeded size limit'); + } + bytes.set(chunk.value, size); + size += chunk.value.byteLength; + } + const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, size)); + return new Response(text, response); + } catch (error) { + cancel(); + throw error; + } finally { + signal?.removeEventListener('abort', cancel); + reader.releaseLock(); + } +} + +export function boundedRepositoryFetch(signal: AbortSignal): typeof fetch { + return async (input, init) => { + signal.throwIfAborted(); + return boundRepositoryResponse( + await fetch(input, { ...init, signal, redirect: 'error' }), + signal + ); + }; +} diff --git a/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.test.ts b/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.test.ts index b3391ef601..9ec7101efe 100644 --- a/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.test.ts +++ b/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.test.ts @@ -1,5 +1,125 @@ -import { describe, expect, it } from '@jest/globals'; -import { BitbucketRepositoryListResultSchema } from './token-service-client'; +jest.mock('@/lib/config.server', () => ({ + GIT_TOKEN_SERVICE_API_URL: 'https://token-service.example', +})); +jest.mock('@/lib/tokens', () => ({ + BITBUCKET_REPOSITORY_LIST_AUDIENCE: 'bitbucket-repository-list', + TOKEN_EXPIRY: { fiveMinutes: 300 }, + generateInternalServiceToken: () => 'service-token', +})); + +import { + BitbucketRepositoryListResultSchema, + fetchBitbucketRepositoriesFromTokenService, + fetchBitbucketWorkspaceAccessTokenRepositoriesFromTokenService, +} from './token-service-client'; + +const fetchMock = jest.spyOn(globalThis, 'fetch'); +const repository = { + id: '12345678-1234-4234-8234-123456789012', + workspaceUuid: '12345678-1234-4234-8234-123456789013', + name: 'repo', + fullName: 'workspace/repo', + private: true, + defaultBranch: 'main', +}; +afterAll(() => fetchMock.mockRestore()); +afterEach(() => jest.useRealTimers()); + +it.each([ + fetchBitbucketRepositoriesFromTokenService, + fetchBitbucketWorkspaceAccessTokenRepositoriesFromTokenService, +])('bounds both authentication paths without changing legacy results', async read => { + fetchMock.mockImplementation(async (_url, init) => { + if (new Headers(init?.headers).get('authorization') !== 'Bearer service-token') + throw new Error('Missing authentication'); + return Response.json({ status: 'available', repositories: Array(51).fill(repository) }); + }); + await expect(read('user', 'organization', { bounded: true })).resolves.toEqual({ + status: 'available', + repositories: Array(50).fill(repository), + }); + await expect(read('user', 'organization')).resolves.toEqual({ + status: 'available', + repositories: Array(51).fill(repository), + }); +}); + +it.each([ + { status: 'available', repositories: [] }, + { status: 'reconnect_required' }, + { status: 'temporarily_unavailable' }, +])('preserves explicit provider states %j', async data => { + fetchMock.mockResolvedValue(Response.json(data)); + await expect( + fetchBitbucketRepositoriesFromTokenService('user', undefined, { bounded: true }) + ).resolves.toEqual(data); +}); + +it.each([ + { status: 'available', repositories: [...Array(50).fill(repository), null] }, + 'invalid json', +])( + 'rejects malformed or oversized bounded data %# but preserves the legacy fallback', + async data => { + fetchMock.mockImplementation(async () => + typeof data === 'string' + ? new Response(data, { headers: { 'content-type': 'application/json' } }) + : Response.json(data) + ); + await expect( + fetchBitbucketRepositoriesFromTokenService('user', undefined, { bounded: true }) + ).rejects.toThrow(); + await expect(fetchBitbucketRepositoriesFromTokenService('user')).resolves.toEqual({ + status: 'temporarily_unavailable', + }); + } +); + +it('rejects valid oversized JSON but keeps the legacy response unchanged', async () => { + const data = { + status: 'available', + repositories: [{ ...repository, name: 'x'.repeat(1048577) }], + }; + fetchMock.mockImplementation(async () => Response.json(data)); + await expect( + fetchBitbucketRepositoriesFromTokenService('user', undefined, { bounded: true }) + ).rejects.toThrow('size limit'); + await expect(fetchBitbucketRepositoriesFromTokenService('user')).resolves.toEqual(data); +}); + +it('keeps network failures retryable without treating them as empty data', async () => { + fetchMock.mockRejectedValue(new Error('network unavailable')); + await expect(fetchBitbucketRepositoriesFromTokenService('user')).resolves.toEqual({ + status: 'temporarily_unavailable', + }); + await expect( + fetchBitbucketRepositoriesFromTokenService('user', undefined, { bounded: true }) + ).rejects.toThrow('network unavailable'); + fetchMock.mockResolvedValue(Response.json({ status: 'available', repositories: [] })); + await expect( + fetchBitbucketRepositoriesFromTokenService('user', undefined, { bounded: true }) + ).resolves.toEqual({ status: 'available', repositories: [] }); +}); + +it('cancels token-service response consumption at the deadline', async () => { + jest.useFakeTimers(); + let cancelled = false; + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { headers: { 'content-type': 'application/json' } } + ) + ); + const result = fetchBitbucketRepositoriesFromTokenService('user', undefined, { bounded: true }); + const rejection = expect(result).rejects.toThrow('Repository fetch timed out'); + await jest.advanceTimersByTimeAsync(30_000); + await rejection; + expect(cancelled).toBe(true); +}); describe('BitbucketRepositoryListResultSchema', () => { it.each(['insufficient_permissions', 'invalid_request'] as const)( diff --git a/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.ts b/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.ts index ad1b01859d..8e6a2e66c9 100644 --- a/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.ts +++ b/apps/web/src/lib/integrations/platforms/bitbucket/token-service-client.ts @@ -1,6 +1,12 @@ import 'server-only'; import { z } from 'zod'; +import { + boundedRepositoryFetch, + REPOSITORY_READ_LIMITS, + withRepositoryReadDeadline, + type RepositoryReadOptions, +} from '@/lib/integrations/core/repository-read-limits'; import { BITBUCKET_CODE_REVIEW_PULL_REQUEST_AUDIENCE, BITBUCKET_CODE_REVIEW_WEBHOOK_DELETE_AUDIENCE, @@ -40,43 +46,64 @@ export type BitbucketRepositoryListResult = z.infer { - if (!GIT_TOKEN_SERVICE_API_URL) return { status: 'temporarily_unavailable' }; - const serviceToken = generateInternalServiceToken(kiloUserId, { - expiresIn: TOKEN_EXPIRY.fiveMinutes, - audience: BITBUCKET_REPOSITORY_LIST_AUDIENCE, - organizationId, - }); - - let response: Response; - try { - response = await fetch(`${GIT_TOKEN_SERVICE_API_URL}/internal/bitbucket/repositories`, { - method: 'POST', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${serviceToken}`, - }, - signal: AbortSignal.timeout(30_000), + return withRepositoryReadDeadline(options, async signal => { + if (!GIT_TOKEN_SERVICE_API_URL) return { status: 'temporarily_unavailable' }; + const serviceToken = generateInternalServiceToken(kiloUserId, { + expiresIn: TOKEN_EXPIRY.fiveMinutes, + audience: BITBUCKET_REPOSITORY_LIST_AUDIENCE, + organizationId, }); - } catch { - return { status: 'temporarily_unavailable' }; - } - if (!response.ok) return { status: 'temporarily_unavailable' }; - try { - const parsed = BitbucketRepositoryListResultSchema.safeParse(await response.json()); - return parsed.success ? parsed.data : { status: 'temporarily_unavailable' }; - } catch { - return { status: 'temporarily_unavailable' }; - } + try { + const response = await (signal ? boundedRepositoryFetch(signal) : fetch)( + `${GIT_TOKEN_SERVICE_API_URL}/internal/bitbucket/repositories`, + { + method: 'POST', + headers: { Accept: 'application/json', Authorization: `Bearer ${serviceToken}` }, + signal: signal ?? AbortSignal.timeout(30_000), + } + ); + if (!response.ok) return { status: 'temporarily_unavailable' }; + const data: unknown = await response.json(); + if ( + signal && + typeof data === 'object' && + data !== null && + 'status' in data && + data.status === 'available' + ) { + const envelope = z + .object({ + status: z.literal('available'), + repositories: z.custom(Array.isArray), + }) + .strict() + .parse(data); + const repositories: BitbucketRepository[] = []; + for (const raw of envelope.repositories) { + const repository = BitbucketRepositorySchema.parse(raw); + if (repositories.length < REPOSITORY_READ_LIMITS.repositories) + repositories.push(repository); + } + return { status: 'available', repositories }; + } + return BitbucketRepositoryListResultSchema.parse(data); + } catch (error) { + if (signal) throw error; + return { status: 'temporarily_unavailable' }; + } + }); } export function fetchBitbucketWorkspaceAccessTokenRepositoriesFromTokenService( kiloUserId: string, - organizationId: string + organizationId: string, + options?: RepositoryReadOptions ): Promise { - return fetchBitbucketRepositoriesFromTokenService(kiloUserId, organizationId); + return fetchBitbucketRepositoriesFromTokenService(kiloUserId, organizationId, options); } const BITBUCKET_CODE_REVIEW_RESPONSE_MAX_BYTES = 256_000; diff --git a/apps/web/src/lib/integrations/platforms/github/adapter.repositories.test.ts b/apps/web/src/lib/integrations/platforms/github/adapter.repositories.test.ts new file mode 100644 index 0000000000..c53ba95d0d --- /dev/null +++ b/apps/web/src/lib/integrations/platforms/github/adapter.repositories.test.ts @@ -0,0 +1,130 @@ +jest.mock('@/lib/utils.server', () => ({ + logExceptInTest: jest.fn(), + warnExceptInTest: jest.fn(), +})); +jest.mock('./app-selector', () => ({ + getGitHubAppCredentials: (app: string) => ({ + appId: app === 'lite' ? '456' : '123', + privateKey: mockPrivateKey, + }), +})); + +import { generateKeyPairSync } from 'node:crypto'; +import { fetchGitHubRepositories } from './adapter'; + +const mockPrivateKey = generateKeyPairSync('rsa', { modulusLength: 2048 }) + .privateKey.export({ type: 'pkcs8', format: 'pem' }) + .toString(); +const repo = { + id: 7, + name: 'repo', + full_name: 'owner/repo', + private: true, + archived: false, + created_at: '2026-01-01T00:00:00Z', +}; +const fetchMock = jest.spyOn(globalThis, 'fetch'); +let pages: string[]; +let appId: string; +let pageResponse: (url: URL) => Response | Promise; + +beforeEach(() => { + pages = []; + appId = ''; + pageResponse = url => + Response.json({ + repositories: Array.from( + { + length: + url.searchParams.get('page') === '3' ? 1 : Number(url.searchParams.get('per_page')), + }, + () => ({ ...repo, archived: url.searchParams.get('page') === '1' }) + ), + }); + fetchMock.mockImplementation(async (input, init) => { + const url = new URL(String(input)); + const authorization = new Headers(init?.headers).get('authorization') ?? ''; + if (url.pathname.endsWith('/access_tokens')) { + appId = JSON.parse(Buffer.from(authorization.split('.')[1], 'base64url').toString()).iss; + return Response.json({ + token: 'installation-token', + expires_at: '2099-01-01T00:00:00Z', + permissions: {}, + repository_selection: 'all', + }); + } + if (authorization !== 'token installation-token') + return Response.json({ message: 'Bad credentials' }, { status: 401 }); + pages.push(url.search); + return pageResponse(url); + }); +}); +afterAll(() => fetchMock.mockRestore()); +afterEach(() => jest.useRealTimers()); + +it.each([undefined, { bounded: true }])( + 'preserves auth and paging with options %j', + async options => { + const result = await fetchGitHubRepositories('42', options ? 'lite' : undefined, options); + expect(result).toHaveLength(options ? 50 : 101); + expect(result[0]).toEqual({ + id: 7, + name: 'repo', + full_name: 'owner/repo', + private: true, + created_at: repo.created_at, + }); + expect(appId).toBe(options ? '456' : '123'); + expect(pages).toHaveLength(options ? 2 : 3); + expect(pages[0]).toContain(`per_page=${options ? 50 : 100}`); + } +); + +it('stops after two archived-only pages', async () => { + pageResponse = () => + Response.json({ + repositories: Array.from({ length: 50 }, () => ({ ...repo, archived: true })), + }); + await expect(fetchGitHubRepositories('42', 'standard', { bounded: true })).resolves.toEqual([]); + expect(pages).toHaveLength(2); +}); + +it.each([ + {}, + { repositories: [null] }, + { repositories: Array(51).fill(repo) }, + 'invalid json', + { repositories: [{ ...repo, name: 'x'.repeat(1048577) }] }, +])('rejects invalid provider data %#', async data => { + pageResponse = () => + typeof data === 'string' + ? new Response(data, { headers: { 'content-type': 'application/json' } }) + : Response.json(data); + await expect(fetchGitHubRepositories('42', 'standard', { bounded: true })).rejects.toThrow(); +}); + +it('preserves provider errors and permits a fresh empty read', async () => { + pageResponse = () => Response.json({ message: 'Provider unavailable' }, { status: 503 }); + await expect(fetchGitHubRepositories('42')).rejects.toThrow('Provider unavailable'); + await expect(fetchGitHubRepositories('42', 'standard', { bounded: true })).rejects.toThrow( + 'Provider unavailable' + ); + pageResponse = () => Response.json({ repositories: [] }); + await expect(fetchGitHubRepositories('42', 'standard', { bounded: true })).resolves.toEqual([]); +}); + +it('includes installation authentication in the deadline', async () => { + jest.useFakeTimers(); + const started = Promise.withResolvers(); + fetchMock.mockImplementation((_url, init) => { + started.resolve(init?.signal as AbortSignal); + return new Promise(() => {}); + }); + const result = fetchGitHubRepositories('42', 'lite', { bounded: true }); + const rejection = expect(result).rejects.toThrow('Repository fetch timed out'); + const signal = await started.promise; + await jest.advanceTimersByTimeAsync(30_000); + await rejection; + expect(signal.aborted).toBe(true); + expect(pages).toEqual([]); +}); diff --git a/apps/web/src/lib/integrations/platforms/github/adapter.ts b/apps/web/src/lib/integrations/platforms/github/adapter.ts index b1d7d93f76..de69346ad1 100644 --- a/apps/web/src/lib/integrations/platforms/github/adapter.ts +++ b/apps/web/src/lib/integrations/platforms/github/adapter.ts @@ -1,3 +1,11 @@ +import { z } from 'zod'; +import { + boundedRepositoryFetch, + repositoryPageSchema, + REPOSITORY_READ_LIMITS, + withRepositoryReadDeadline, + type RepositoryReadOptions, +} from '@/lib/integrations/core/repository-read-limits'; import { Octokit } from '@octokit/rest'; import { createAppAuth } from '@octokit/auth-app'; import { exchangeWebFlowCode } from '@octokit/oauth-methods'; @@ -66,7 +74,8 @@ export function verifyGitHubWebhookSignature( */ export async function generateGitHubInstallationToken( installationId: string, - appType: GitHubAppType = 'standard' + appType: GitHubAppType = 'standard', + request?: Octokit['request'] ): Promise { const credentials = getGitHubAppCredentials(appType); @@ -78,6 +87,7 @@ export async function generateGitHubInstallationToken( appId: credentials.appId, privateKey: credentials.privateKey, installationId, + ...(request ? { request } : {}), }); const authResult = await auth({ type: 'installation' }); @@ -136,40 +146,66 @@ type GitHubBranch = { */ export async function fetchGitHubRepositories( installationId: string, - appType: GitHubAppType = 'standard' + appType: GitHubAppType = 'standard', + options?: RepositoryReadOptions ): Promise { - const tokenData = await generateGitHubInstallationToken(installationId, appType); - const octokit = new Octokit({ auth: tokenData.token }); - - // Fetch all repositories accessible by the installation using pagination - const repositories: GitHubRepository[] = []; - let page = 1; - const perPage = 100; - - while (true) { - const { data } = await octokit.apps.listReposAccessibleToInstallation({ - per_page: 100, - page, - }); - - // Filter out archived repositories - repositories.push( - ...data.repositories - .filter(repo => !repo.archived) - .map(repo => ({ + return withRepositoryReadDeadline(options, async signal => { + const request = signal ? { fetch: boundedRepositoryFetch(signal), signal } : undefined; + const tokenData = await generateGitHubInstallationToken( + installationId, + appType, + request ? new Octokit({ request }).request : undefined + ); + const octokit = new Octokit({ auth: tokenData.token, ...(request ? { request } : {}) }); + const repositories: GitHubRepository[] = []; + let page = 1; + const perPage = signal ? REPOSITORY_READ_LIMITS.repositories : 100; + + while (!signal || page <= REPOSITORY_READ_LIMITS.pages) { + const result = await octokit.apps.listReposAccessibleToInstallation({ + per_page: perPage, + page, + }); + const data = signal + ? z + .object({ + repositories: repositoryPageSchema.pipe( + z.array( + z.object({ + id: z.number(), + name: z.string(), + full_name: z.string(), + private: z.boolean(), + archived: z.boolean(), + created_at: z.string().nullish(), + }) + ) + ), + }) + .parse(result.data) + : result.data; + const active = data.repositories.filter(repo => !repo.archived); + const selected = signal + ? active.slice(0, REPOSITORY_READ_LIMITS.repositories - repositories.length) + : active; + repositories.push( + ...selected.map(repo => ({ id: repo.id, name: repo.name, full_name: repo.full_name, private: repo.private, created_at: repo.created_at ?? new Date().toISOString(), })) - ); - - if (data.repositories.length < perPage) break; - page++; - } - - return repositories; + ); + if ( + data.repositories.length < perPage || + (signal && repositories.length >= REPOSITORY_READ_LIMITS.repositories) + ) + break; + page++; + } + return repositories; + }); } /** diff --git a/apps/web/src/lib/integrations/platforms/gitlab/adapter.test.ts b/apps/web/src/lib/integrations/platforms/gitlab/adapter.test.ts index d87ad3214b..7b805a3c84 100644 --- a/apps/web/src/lib/integrations/platforms/gitlab/adapter.test.ts +++ b/apps/web/src/lib/integrations/platforms/gitlab/adapter.test.ts @@ -1,3 +1,7 @@ +jest.mock('@/lib/utils.server', () => ({ logExceptInTest: jest.fn() })); +jest.mock('@/lib/integrations/oauth/urls', () => ({ + getPlatformOAuthCallbackUrl: () => 'https://app.example/callback', +})); jest.mock('dns/promises', () => ({ lookup: jest.fn(), })); @@ -426,6 +430,117 @@ describe('fetchGitLabProjects', () => { }); }); +describe('bounded GitLab projects', () => { + beforeEach(() => mockFetch.mockReset()); + afterEach(() => jest.useRealTimers()); + const active = createGitLabProjectDiscoveryResponse()[0]; + + it.each([false, true])('bounds archived-only pages, self-hosted=%s', async selfHosted => { + const json = Array(50).fill({ ...active, archived: true }); + if (selfHosted) { + mockSelfHostedGitLabResponse({ status: 200, json }); + mockSelfHostedGitLabResponse({ status: 200, json }); + } else mockFetch.mockImplementation(async () => Response.json(json)); + await expect( + fetchGitLabProjects('token', selfHosted ? 'https://gitlab.example.com' : undefined, { + bounded: true, + }) + ).resolves.toEqual([]); + expect(selfHosted ? mockHttpsRequest : mockFetch).toHaveBeenCalledTimes(2); + }); + + it('caps accumulation across next-page jumps', async () => { + mockFetch + .mockResolvedValueOnce( + Response.json(Array(40).fill(active), { headers: { 'x-next-page': '8' } }) + ) + .mockResolvedValueOnce( + Response.json(Array(40).fill(active), { headers: { 'x-next-page': '99' } }) + ); + const result = await fetchGitLabProjects('token', undefined, { bounded: true }); + expect(result).toEqual( + Array(50).fill({ + id: 123, + name: 'active-project', + full_name: 'group/active-project', + private: true, + }) + ); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it.each([ + {}, + [null], + Array(51).fill(active), + 'invalid json', + [{ ...active, name: 'x'.repeat(1048577) }], + ])('rejects malformed or oversized pages %#', async json => { + mockFetch.mockResolvedValue( + typeof json === 'string' + ? new Response(json, { headers: { 'content-type': 'application/json' } }) + : Response.json(json) + ); + await expect(fetchGitLabProjects('token', undefined, { bounded: true })).rejects.toThrow(); + }); + + it('rejects self-hosted bytes before the legacy 10 MiB ceiling', async () => { + mockSelfHostedGitLabResponse({ status: 200, body: 'x'.repeat(1048577) }); + await expect( + fetchGitLabProjects('token', 'https://gitlab.example.com', { bounded: true }) + ).rejects.toThrow('GitLab response exceeded size limit'); + }); + + it('keeps DNS binding and strips credentials across a public-to-self-hosted redirect', async () => { + mockFetch.mockResolvedValue( + new Response(null, { + status: 302, + headers: { location: 'https://gitlab.example.com/api/v4/projects' }, + }) + ); + mockSelfHostedGitLabResponse({ status: 200, json: [] }); + await expect(fetchGitLabProjects('token', undefined, { bounded: true })).resolves.toEqual([]); + const options = mockHttpsRequest.mock.calls[0][0]; + expect(options.headers.authorization).toBeUndefined(); + const resolved = jest.fn(); + options.lookup('gitlab.example.com', {}, resolved); + expect(resolved).toHaveBeenCalledWith(null, '93.184.216.34', 4); + }); + + it('rejects unsafe redirect destinations without returning repositories', async () => { + mockFetch.mockResolvedValue( + new Response(null, { + status: 302, + headers: { location: 'https://127.0.0.1/api/v4/projects' }, + }) + ); + await expect(fetchGitLabProjects('token', undefined, { bounded: true })).rejects.toThrow( + 'host is not allowed' + ); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('uses one deadline across pages and cancels the pending transport', async () => { + jest.useFakeTimers(); + let signal: AbortSignal | undefined; + mockFetch.mockImplementation((_url, init) => { + signal = init.signal; + return new Promise(resolve => + setTimeout( + () => resolve(Response.json(Array(50).fill({ ...active, archived: true }))), + 20_000 + ) + ); + }); + const result = fetchGitLabProjects('token', undefined, { bounded: true }); + const rejection = expect(result).rejects.toThrow('Repository fetch timed out'); + await jest.advanceTimersByTimeAsync(30_000); + await rejection; + expect(signal?.aborted).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); +}); + describe('fetchGitLabRootTextFileAtRef', () => { beforeEach(() => { mockFetch.mockReset(); diff --git a/apps/web/src/lib/integrations/platforms/gitlab/adapter.ts b/apps/web/src/lib/integrations/platforms/gitlab/adapter.ts index cae231422d..d138a24bf8 100644 --- a/apps/web/src/lib/integrations/platforms/gitlab/adapter.ts +++ b/apps/web/src/lib/integrations/platforms/gitlab/adapter.ts @@ -5,6 +5,14 @@ * Supports both GitLab.com and self-hosted GitLab instances. */ +import { z } from 'zod'; +import { + boundRepositoryResponse, + repositoryPageSchema, + REPOSITORY_READ_LIMITS, + withRepositoryReadDeadline, + type RepositoryReadOptions, +} from '@/lib/integrations/core/repository-read-limits'; import { getEnvVariable } from '@/lib/dotenvx'; import { PLATFORM } from '@/lib/integrations/core/constants'; import type { PlatformRepository } from '@/lib/integrations/core/types'; @@ -32,8 +40,16 @@ const MAX_GITLAB_REDIRECTS = 5; const MAX_GITLAB_RESPONSE_BYTES = 10 * 1024 * 1024; const GITLAB_REQUEST_TIMEOUT_MS = 30_000; -async function fetchGitLab(url: string, init?: RequestInit, redirectCount = 0): Promise { - const response = await fetchGitLabOnce(url, init); +async function fetchGitLab( + url: string, + init?: RequestInit, + redirectCount = 0, + bounded = false +): Promise { + const rawResponse = await fetchGitLabOnce(url, init, bounded); + const response = bounded + ? await boundRepositoryResponse(rawResponse, init?.signal ?? undefined) + : rawResponse; if (!isGitLabRedirect(response.status)) { return response; } @@ -51,17 +67,24 @@ async function fetchGitLab(url: string, init?: RequestInit, redirectCount = 0): return fetchGitLab( redirectUrl, buildRedirectRequestInit(init, response.status, url, redirectUrl), - redirectCount + 1 + redirectCount + 1, + bounded ); } -async function fetchGitLabOnce(url: string, init?: RequestInit): Promise { +async function fetchGitLabOnce( + url: string, + init?: RequestInit, + bounded = false +): Promise { + if (bounded) init?.signal?.throwIfAborted(); const resolvedUrl = await resolveGitLabUrlSafely(url); + if (bounded) init?.signal?.throwIfAborted(); if (!resolvedUrl.address) { return fetch(url, { ...init, redirect: 'manual' }); } - return fetchGitLabBoundToAddress({ ...resolvedUrl, address: resolvedUrl.address }, init); + return fetchGitLabBoundToAddress({ ...resolvedUrl, address: resolvedUrl.address }, init, bounded); } function isGitLabRedirect(status: number): boolean { @@ -109,8 +132,10 @@ function buildRedirectRequestInit( function fetchGitLabBoundToAddress( { url, address, family }: GitLabResolvedUrl & { address: string }, - init?: RequestInit + init?: RequestInit, + bounded = false ): Promise { + const maxBytes = bounded ? REPOSITORY_READ_LIMITS.responseBytes : MAX_GITLAB_RESPONSE_BYTES; const request = url.protocol === 'https:' ? https.request : http.request; const headers = headersInitToRecord(init?.headers); const body = bodyInitToBuffer(init?.body); @@ -138,7 +163,7 @@ function fetchGitLabBoundToAddress( response.on('data', chunk => { const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); responseBytes += buffer.byteLength; - if (responseBytes > MAX_GITLAB_RESPONSE_BYTES) { + if (responseBytes > maxBytes) { const error = new Error('GitLab response exceeded size limit'); response.destroy(error); req.destroy(error); @@ -286,7 +311,9 @@ export type GitLabProject = { marked_for_deletion_at?: string | null; }; -function isActiveGitLabProject(project: GitLabProject): boolean { +function isActiveGitLabProject( + project: Pick +): boolean { return !project.archived && !project.marked_for_deletion_on && !project.marked_for_deletion_at; } @@ -436,57 +463,80 @@ export async function fetchGitLabUser( */ export async function fetchGitLabProjects( accessToken: string, - instanceUrl: string = DEFAULT_GITLAB_URL + instanceUrl: string = DEFAULT_GITLAB_URL, + options?: RepositoryReadOptions ): Promise { - const projects: PlatformRepository[] = []; - let page = 1; - const perPage = 100; - - while (true) { - const response = await fetchGitLab( - buildGitLabUrl(instanceUrl, '/api/v4/projects', { - membership: true, - per_page: perPage, - page, - archived: false, - }), - { - headers: { - Authorization: `Bearer ${accessToken}`, + return withRepositoryReadDeadline(options, async signal => { + const projects: PlatformRepository[] = []; + let page = 1; + let fetchedPages = 0; + const perPage = signal ? REPOSITORY_READ_LIMITS.repositories : 100; + + while (!signal || fetchedPages < REPOSITORY_READ_LIMITS.pages) { + fetchedPages++; + const response = await fetchGitLab( + buildGitLabUrl(instanceUrl, '/api/v4/projects', { + membership: true, + per_page: perPage, + page, + archived: false, + }), + { + headers: { + Authorization: `Bearer ${accessToken}`, + ...(signal ? { Accept: 'application/json' } : {}), + }, + ...(signal ? { signal } : {}), }, + 0, + !!signal + ); + if (!response.ok) { + const error = await response.text(); + logExceptInTest('GitLab projects fetch failed:', { status: response.status, error }); + throw new Error(`GitLab projects fetch failed: ${response.status}`); } - ); - - if (!response.ok) { - const error = await response.text(); - logExceptInTest('GitLab projects fetch failed:', { status: response.status, error }); - throw new Error(`GitLab projects fetch failed: ${response.status}`); - } - - const data = (await response.json()) as GitLabProject[]; - - projects.push( - ...data.filter(isActiveGitLabProject).map(project => ({ - id: project.id, - name: project.name, - full_name: project.path_with_namespace, - private: project.visibility === 'private', - })) - ); - - const nextPage = Number.parseInt(response.headers.get('x-next-page') || '', 10); - if (Number.isInteger(nextPage) && nextPage > page) { - page = nextPage; - continue; + const data = signal + ? repositoryPageSchema + .pipe( + z.array( + z.object({ + id: z.number(), + name: z.string(), + path_with_namespace: z.string(), + visibility: z.enum(['private', 'internal', 'public']), + archived: z.boolean(), + marked_for_deletion_on: z.string().nullish(), + marked_for_deletion_at: z.string().nullish(), + }) + ) + ) + .parse(await response.json()) + : ((await response.json()) as GitLabProject[]); + const active = data.filter(isActiveGitLabProject); + const selected = signal + ? active.slice(0, REPOSITORY_READ_LIMITS.repositories - projects.length) + : active; + projects.push( + ...selected.map(project => ({ + id: project.id, + name: project.name, + full_name: project.path_with_namespace, + private: project.visibility === 'private', + })) + ); + if (signal && projects.length >= REPOSITORY_READ_LIMITS.repositories) break; + const nextPage = Number.parseInt(response.headers.get('x-next-page') || '', 10); + if (Number.isInteger(nextPage) && nextPage > page) { + page = nextPage; + continue; + } + if (data.length < perPage) break; + page++; } - - if (data.length < perPage) break; - page++; - } - - logExceptInTest('GitLab projects fetched', { count: projects.length }); - - return projects; + logExceptInTest('GitLab projects fetched', { count: projects.length }); + return projects; + }); } /**