diff --git a/src/index.ts b/src/index.ts index 81aa335..03e12f5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,15 @@ export { type Uploadable, toFile } from './core/uploads'; export { APIPromise } from './core/api-promise'; export { Kernel, type ClientOptions } from './client'; export { type BrowserFetchInit } from './lib/browser-fetch'; +export { + AuditLogDownloadError, + type AuditLogDownloadDestination, + type AuditLogDownloadOptions, + type AuditLogDownloadParams, + type AuditLogDownloadProgress, + type AuditLogDownloadResult, + type AuditLogDownloadWriteResult, +} from './lib/audit-log-download'; export { BrowserRouteCache, type BrowserRoute } from './lib/browser-routing'; export { PagePromise } from './core/pagination'; export { diff --git a/src/lib/audit-log-download.ts b/src/lib/audit-log-download.ts new file mode 100644 index 0000000..b92383d --- /dev/null +++ b/src/lib/audit-log-download.ts @@ -0,0 +1,234 @@ +import { APIConnectionTimeoutError, APIUserAbortError, KernelError } from '../core/error'; +import type { RequestOptions } from '../internal/request-options'; +import type { AuditLogExportChunkParams } from '../resources/audit-logs'; + +const DEFAULT_MAX_TRANSFER_RETRIES = 6; +const MAX_CHUNK_ROWS = 50_000; +const MAX_RETRY_DELAY_MS = 8_000; + +export class AuditLogDownloadError extends KernelError {} + +export type AuditLogDownloadParams = Omit; + +export interface AuditLogDownloadResult { + bytesWritten: number; + chunks: number; + rows: number; +} + +export interface AuditLogDownloadProgress extends AuditLogDownloadResult { + chunkRows: number; +} + +export type AuditLogDownloadWriteResult = void | number | { bytesWritten: number }; + +export interface AuditLogDownloadDestination { + write(chunk: Uint8Array): AuditLogDownloadWriteResult | Promise; +} + +export interface AuditLogDownloadOptions + extends Omit< + RequestOptions, + 'method' | 'path' | 'query' | 'body' | 'stream' | '__binaryResponse' | '__streamClass' + > { + onProgress?(progress: AuditLogDownloadProgress): void | Promise; + maxTransferRetries?: number; +} + +type FetchChunk = (query: AuditLogExportChunkParams, options?: RequestOptions) => Promise; + +export async function downloadAuditLogs( + fetchChunk: FetchChunk, + query: AuditLogDownloadParams, + destination: AuditLogDownloadDestination, + defaultTimeout: number, + options: AuditLogDownloadOptions = {}, +): Promise { + if (!destination || typeof destination.write !== 'function') { + throw new TypeError('audit log download destination must provide write()'); + } + + const { onProgress, maxTransferRetries = DEFAULT_MAX_TRANSFER_RETRIES, ...requestOptions } = options; + if (!Number.isInteger(maxTransferRetries) || maxTransferRetries < 0) { + throw new TypeError('maxTransferRetries must be a non-negative integer'); + } + const timeout = requestOptions.timeout ?? defaultTimeout; + let cursor: string | undefined; + const result: AuditLogDownloadResult = { bytesWritten: 0, chunks: 0, rows: 0 }; + const seenCursors = new Set(); + + while (true) { + const chunk = await fetchVerifiedChunk( + fetchChunk, + cursor ? { ...query, cursor } : query, + requestOptions, + maxTransferRetries, + timeout, + ); + const { nextCursor, hasMore, rows } = parseChunkHeaders(chunk.headers, cursor); + if (hasMore && nextCursor) { + if (seenCursors.has(nextCursor)) { + throw new AuditLogDownloadError('response repeated X-Next-Cursor header'); + } + seenCursors.add(nextCursor); + } + await writeChunk(destination, chunk.body); + + cursor = nextCursor; + result.bytesWritten += chunk.body.byteLength; + result.chunks += 1; + result.rows += rows; + if (onProgress) { + await onProgress({ ...result, chunkRows: rows }); + } + if (!hasMore) { + return result; + } + } +} + +async function fetchVerifiedChunk( + fetchChunk: FetchChunk, + query: AuditLogExportChunkParams, + options: RequestOptions, + maxTransferRetries: number, + timeout: number, +): Promise<{ body: Uint8Array; headers: Headers }> { + for (let retries = 0; ; retries += 1) { + const controller = new AbortController(); + const onAbort = () => controller.abort(); + if (options.signal?.aborted) { + controller.abort(); + } else { + options.signal?.addEventListener('abort', onAbort, { once: true }); + } + + let response: Response; + try { + response = await fetchChunk(query, { ...options, signal: controller.signal }); + } catch (error) { + options.signal?.removeEventListener('abort', onAbort); + throw error; + } + + let bodyTimedOut = false; + const timer = setTimeout(() => { + bodyTimedOut = true; + controller.abort(); + }, timeout); + try { + const body = new Uint8Array(await response.arrayBuffer()); + clearTimeout(timer); + if (options.signal?.aborted) { + throw new APIUserAbortError(); + } + if (bodyTimedOut) { + throw new APIConnectionTimeoutError(); + } + const expected = response.headers.get('x-content-sha256'); + if (!expected) { + throw new AuditLogDownloadError('response missing X-Content-Sha256 header'); + } + const actual = await sha256Hex(body); + if (options.signal?.aborted) { + throw new APIUserAbortError(); + } + if (actual !== expected) { + throw new AuditLogDownloadError( + `audit log chunk checksum mismatch (got ${actual}, want ${expected})`, + ); + } + return { body, headers: response.headers }; + } catch (error) { + if (options.signal?.aborted && !(error instanceof APIUserAbortError)) { + error = new APIUserAbortError(); + } else if (bodyTimedOut) { + error = new APIConnectionTimeoutError(); + } + if (retries === maxTransferRetries || error instanceof APIUserAbortError) { + throw error; + } + await retryDelay(retries + 1, options.signal); + } finally { + clearTimeout(timer); + options.signal?.removeEventListener('abort', onAbort); + } + } +} + +function parseChunkHeaders( + headers: Headers, + currentCursor: string | undefined, +): { rows: number; nextCursor: string | undefined; hasMore: boolean } { + const hasMoreValue = headers.get('x-has-more'); + if (hasMoreValue !== 'true' && hasMoreValue !== 'false') { + throw new AuditLogDownloadError('response missing or invalid X-Has-More header'); + } + const hasMore = hasMoreValue === 'true'; + + const rowCount = headers.get('x-row-count'); + if (rowCount === null || !/^[0-9]+$/.test(rowCount)) { + throw new AuditLogDownloadError('response missing or invalid X-Row-Count header'); + } + const rows = Number(rowCount); + if (!Number.isSafeInteger(rows) || rows > MAX_CHUNK_ROWS) { + throw new AuditLogDownloadError('response missing or invalid X-Row-Count header'); + } + + const nextCursor = headers.get('x-next-cursor') || undefined; + if (hasMore && (!nextCursor || nextCursor === currentCursor)) { + throw new AuditLogDownloadError('response has invalid X-Next-Cursor header'); + } + if (!hasMore && nextCursor) { + throw new AuditLogDownloadError('response returned a cursor after the final chunk'); + } + return { rows, nextCursor, hasMore }; +} + +async function sha256Hex(body: Uint8Array): Promise { + const digest = await globalThis.crypto.subtle.digest('SHA-256', body); + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join(''); +} + +async function retryDelay(attempt: number, signal: AbortSignal | null | undefined): Promise { + const delay = Math.min(1_000 * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS); + await new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new APIUserAbortError()); + return; + } + const onAbort = () => { + clearTimeout(timer); + reject(new APIUserAbortError()); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, delay); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +async function writeChunk(destination: AuditLogDownloadDestination, body: Uint8Array): Promise { + let offset = 0; + while (offset < body.byteLength) { + const result = await destination.write(offset === 0 ? body : body.subarray(offset)); + if (typeof result === 'number') { + offset += validateWriteCount(result, body.byteLength - offset); + continue; + } + if (result && typeof result === 'object' && 'bytesWritten' in result) { + const bytesWritten = (result as { bytesWritten: number }).bytesWritten; + offset += validateWriteCount(bytesWritten, body.byteLength - offset); + continue; + } + return; + } +} + +function validateWriteCount(value: number, remaining: number): number { + if (!Number.isSafeInteger(value) || value <= 0 || value > remaining) { + throw new AuditLogDownloadError('audit log download destination performed a short write'); + } + return value; +} diff --git a/src/resources/audit-logs.ts b/src/resources/audit-logs.ts index 9367dc9..ca7dd08 100644 --- a/src/resources/audit-logs.ts +++ b/src/resources/audit-logs.ts @@ -5,6 +5,22 @@ import { APIPromise } from '../core/api-promise'; import { PagePromise, PageTokenPagination, type PageTokenPaginationParams } from '../core/pagination'; import { buildHeaders } from '../internal/headers'; import { RequestOptions } from '../internal/request-options'; +import { + downloadAuditLogs, + type AuditLogDownloadDestination, + type AuditLogDownloadOptions, + type AuditLogDownloadParams, + type AuditLogDownloadProgress, + type AuditLogDownloadResult, +} from '../lib/audit-log-download'; + +export type { + AuditLogDownloadDestination, + AuditLogDownloadOptions, + AuditLogDownloadParams, + AuditLogDownloadProgress, + AuditLogDownloadResult, +} from '../lib/audit-log-download'; /** * Read audit log records for the authenticated organization. @@ -34,6 +50,27 @@ export class AuditLogs extends APIResource { __binaryResponse: true, }); } + + /** + * Download a complete gzip-compressed JSON Lines audit log export to a writable + * destination. The SDK verifies every chunk and retries transient transfer + * failures. It does not close the destination. If the download fails, the + * destination may contain a partial export; use a temporary file and atomic + * rename when the completed export must be published atomically. + */ + download( + query: AuditLogDownloadParams, + destination: AuditLogDownloadDestination, + options?: AuditLogDownloadOptions, + ): Promise { + return downloadAuditLogs( + (chunkQuery, chunkOptions) => this.exportChunk(chunkQuery, chunkOptions), + query, + destination, + this._client.timeout, + options, + ); + } } export type AuditLogEntriesPageTokenPagination = PageTokenPagination; @@ -205,5 +242,10 @@ export declare namespace AuditLogs { type AuditLogEntriesPageTokenPagination as AuditLogEntriesPageTokenPagination, type AuditLogListParams as AuditLogListParams, type AuditLogExportChunkParams as AuditLogExportChunkParams, + type AuditLogDownloadDestination as AuditLogDownloadDestination, + type AuditLogDownloadOptions as AuditLogDownloadOptions, + type AuditLogDownloadParams as AuditLogDownloadParams, + type AuditLogDownloadProgress as AuditLogDownloadProgress, + type AuditLogDownloadResult as AuditLogDownloadResult, }; } diff --git a/tests/lib/audit-log-download.test.ts b/tests/lib/audit-log-download.test.ts new file mode 100644 index 0000000..4594686 --- /dev/null +++ b/tests/lib/audit-log-download.test.ts @@ -0,0 +1,334 @@ +import { createHash } from 'node:crypto'; +import { open, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import Kernel, { APIConnectionTimeoutError, APIUserAbortError } from '@onkernel/sdk'; + +const checksum = (body: string) => createHash('sha256').update(body).digest('hex'); + +const chunkResponse = (body: string, rows: number, hasMore: boolean, nextCursor?: string) => + new Response(body, { + headers: { + 'content-type': 'application/octet-stream', + 'x-content-sha256': checksum(body), + 'x-has-more': String(hasMore), + 'x-row-count': String(rows), + ...(nextCursor ? { 'x-next-cursor': nextCursor } : {}), + }, + }); + +const requestURL = (input: string | URL | Request) => + typeof input === 'string' ? input + : input instanceof URL ? input.toString() + : input.url; + +const stalledChunkResponse = (signal: AbortSignal) => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('partial')); + signal.addEventListener( + 'abort', + () => controller.error(new DOMException('This operation was aborted', 'AbortError')), + { once: true }, + ); + }, + }), + { + headers: { + 'x-content-sha256': checksum('partial'), + 'x-has-more': 'false', + 'x-row-count': '1', + }, + }, + ); + +describe('audit log download', () => { + test('writes verified chunks and reports progress', async () => { + const cursors: Array = []; + const formats: Array = []; + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async (input) => { + const url = new URL(requestURL(input)); + cursors.push(url.searchParams.get('cursor')); + formats.push(url.searchParams.get('format')); + return cursors.length === 1 ? + chunkResponse('first', 2, true, 'next') + : chunkResponse('second', 1, false); + }, + }); + const chunks: Uint8Array[] = []; + const progress: Array<{ bytesWritten: number; chunks: number; rows: number; chunkRows: number }> = []; + + const result = await client.auditLogs.download( + { + start: '2026-06-01T00:00:00Z', + end: '2026-06-02T00:00:00Z', + }, + { + write(chunk) { + chunks.push(chunk); + }, + }, + { + onProgress(update) { + progress.push(update); + }, + }, + ); + + expect(cursors).toEqual([null, 'next']); + expect(formats).toEqual([null, null]); + expect(Buffer.concat(chunks).toString()).toBe('firstsecond'); + expect(result).toEqual({ bytesWritten: 11, chunks: 2, rows: 3 }); + expect(progress).toEqual([ + { bytesWritten: 5, chunks: 1, rows: 2, chunkRows: 2 }, + { bytesWritten: 11, chunks: 2, rows: 3, chunkRows: 1 }, + ]); + }); + + test('writes to a Node file handle', async () => { + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => chunkResponse('chunk', 1, false), + }); + const directory = await mkdtemp(join(tmpdir(), 'kernel-audit-logs-')); + const path = join(directory, 'audit-logs.jsonl.gz'); + + try { + const file = await open(path, 'w'); + try { + await client.auditLogs.download({ start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, file); + } finally { + await file.close(); + } + expect(await readFile(path, 'utf8')).toBe('chunk'); + } finally { + await rm(directory, { recursive: true }); + } + }); + + test('rejects an invalid next cursor before writing', async () => { + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => chunkResponse('chunk', 1, true), + }); + const write = jest.fn(); + + await expect( + client.auditLogs.download({ start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, { write }), + ).rejects.toThrow('response has invalid X-Next-Cursor header'); + expect(write).not.toHaveBeenCalled(); + }); + + test.each(['', '1.0', '50001'])('rejects invalid row count %p', async (rowCount) => { + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => { + const response = chunkResponse('chunk', 1, false); + response.headers.set('x-row-count', rowCount); + return response; + }, + }); + const write = jest.fn(); + + await expect( + client.auditLogs.download({ start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, { write }), + ).rejects.toThrow('response missing or invalid X-Row-Count header'); + expect(write).not.toHaveBeenCalled(); + }); + + test.each([NaN, 0.5, { bytesWritten: NaN }, { bytesWritten: 0.5 }])( + 'rejects invalid destination write result %p', + async (writeResult) => { + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => chunkResponse('chunk', 1, false), + }); + + await expect( + client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write: () => writeResult }, + ), + ).rejects.toThrow('audit log download destination performed a short write'); + }, + ); + + test('retries a checksum mismatch without writing the bad chunk', async () => { + let attempts = 0; + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => { + attempts += 1; + if (attempts === 1) { + return new Response('bad', { + headers: { + 'x-content-sha256': checksum('good'), + 'x-has-more': 'false', + 'x-row-count': '1', + }, + }); + } + return chunkResponse('good', 1, false); + }, + }); + const chunks: Uint8Array[] = []; + + const download = client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { + write(chunk) { + chunks.push(chunk); + }, + }, + ); + await download; + + expect(attempts).toBe(2); + expect(Buffer.concat(chunks).toString()).toBe('good'); + }); + + test('respects request retry options', async () => { + let attempts = 0; + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => { + attempts += 1; + return new Response(JSON.stringify({ message: 'temporary failure' }), { + status: 500, + headers: { 'content-type': 'application/json', 'x-should-retry': 'false' }, + }); + }, + }); + + await expect( + client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write() {} }, + { maxRetries: 0 }, + ), + ).rejects.toThrow('500'); + expect(attempts).toBe(1); + }); + + test('respects transfer retry options', async () => { + let attempts = 0; + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => { + attempts += 1; + return new Response('bad', { + headers: { + 'x-content-sha256': checksum('good'), + 'x-has-more': 'false', + 'x-row-count': '1', + }, + }); + }, + }); + + await expect( + client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write() {} }, + { maxTransferRetries: 0 }, + ), + ).rejects.toThrow('checksum mismatch'); + expect(attempts).toBe(1); + }); + + test('times out a stalled response body', async () => { + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async (_input, init) => stalledChunkResponse(init?.signal as AbortSignal), + }); + + await expect( + client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write() {} }, + { timeout: 10, maxTransferRetries: 0 }, + ), + ).rejects.toBeInstanceOf(APIConnectionTimeoutError); + }); + + test('does not apply the body timeout to checksum verification', async () => { + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => chunkResponse('chunk', 1, false), + }); + const digest = globalThis.crypto.subtle.digest.bind(globalThis.crypto.subtle); + const digestSpy = jest + .spyOn(globalThis.crypto.subtle, 'digest') + .mockImplementation(async (algorithm, data) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return digest(algorithm, data); + }); + + try { + await expect( + client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write() {} }, + { timeout: 10, maxTransferRetries: 0 }, + ), + ).resolves.toEqual({ bytesWritten: 5, chunks: 1, rows: 1 }); + } finally { + digestSpy.mockRestore(); + } + }); + + test('maps cancellation during a response body read to APIUserAbortError', async () => { + const controller = new AbortController(); + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async (_input, init) => stalledChunkResponse(init?.signal as AbortSignal), + }); + + const download = client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write() {} }, + { signal: controller.signal, maxTransferRetries: 0 }, + ); + setTimeout(() => controller.abort(), 0); + + await expect(download).rejects.toBeInstanceOf(APIUserAbortError); + }); + + test('rejects a cursor cycle before writing duplicate data', async () => { + let attempts = 0; + const client = new Kernel({ + apiKey: 'test', + baseURL: 'https://api.example', + fetch: async () => { + attempts += 1; + if (attempts === 1) return chunkResponse('first', 1, true, 'a'); + if (attempts === 2) return chunkResponse('second', 1, true, 'b'); + return chunkResponse('duplicate', 1, true, 'a'); + }, + }); + const chunks: Uint8Array[] = []; + + await expect( + client.auditLogs.download( + { start: '2026-06-01T00:00:00Z', end: '2026-06-02T00:00:00Z' }, + { write: (chunk) => void chunks.push(chunk) }, + ), + ).rejects.toThrow('response repeated X-Next-Cursor header'); + expect(Buffer.concat(chunks).toString()).toBe('firstsecond'); + }); +});