diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d1c5572..02cc325 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,7 @@ jobs: strategy: matrix: - node-version: ["16.x", "18.x", "20.x"] + node-version: ["18.x", "20.x", "22.x"] steps: - uses: actions/checkout@v4 diff --git a/.npmignore b/.npmignore index 8eba6c8..ac4d7bb 100644 --- a/.npmignore +++ b/.npmignore @@ -1 +1,2 @@ src/ +scripts/ diff --git a/README.md b/README.md index c0add65..38a2aad 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,105 @@ Signup at [Urlbox](https://urlbox.com) to get an API key and secret. npm install urlbox --save ``` -## Example +## API client + +`Urlbox` calls the render API directly. It needs Node 18+ (or any runtime with a global `fetch`; pass your own as `fetch` otherwise) and both your API key and secret, so use it server-side only. + +```ts +import { Urlbox } from "urlbox"; + +const urlbox = new Urlbox({ + apiKey: process.env.URLBOX_API_KEY!, + secret: process.env.URLBOX_SECRET!, +}); + +// Render and wait for the result (POST /v1/render/sync) +const result = await urlbox.render({ + url: "https://example.com", + format: "png", + full_page: true, +}); +console.log(result.renderUrl, result.size); + +// Queue a render (POST /v1/render/async) and poll for it +const { renderId } = await urlbox.renderAsync({ + url: "https://example.com", + webhook_url: "https://yourapp.com/webhooks/urlbox", +}); +const status = await urlbox.getRender(renderId); // created | retrying | succeeded | failed | not-found +const done = await urlbox.waitForRender(renderId, { intervalMs: 2000, timeoutMs: 120_000 }); + +// Signed render link, the same as the default export's generateRenderLink +const link = urlbox.generateRenderLink({ url: "https://example.com" }); +``` + +Constructor options: `apiKey`, `secret`, `baseUrl` (default `https://api.urlbox.com`), `fetch`, and `timeoutMs` (per request, default 120000). + +`RenderOptions` is generated from the API's option schema, so every option is typed and documented in your editor. See [urlbox.com/docs](https://urlbox.com/docs) for the full reference. + +### Errors + +Failed requests throw `UrlboxError`: + +```ts +import { UrlboxError, UrlboxRenderFailedError } from "urlbox"; + +try { + await urlbox.render({ url: "https://example.com" }); +} catch (err) { + if (err instanceof UrlboxError) { + err.status; // HTTP status (0 for network errors and timeouts) + err.message; // from the x-urlbox-error-message header or the body + err.code; // e.g. "InvalidOptions" + err.requestId; // x-urlbox-request-id, quote it to support + err.rateLimit; // { limit, remaining, reset, retryAfter } when present + } +} +``` + +`waitForRender` throws `UrlboxRenderFailedError` (a subclass of `UrlboxError`) when the render fails, and a `UrlboxError` with code `WAIT_TIMEOUT` if it does not finish in time. + +## Verifying webhooks + +Urlbox signs every webhook with the `X-Urlbox-Signature` header: `t=,sha256=`, with one `sha256` entry per active secret while you rotate. Verify it against the **raw request body**, exactly as received. Re-serialising parsed JSON will not match. + +```ts +import express from "express"; +import { parseWebhook, UrlboxWebhookSignatureError } from "urlbox"; + +const app = express(); + +app.post( + "/webhooks/urlbox", + express.raw({ type: "application/json" }), // keep the body as a Buffer + (req, res) => { + try { + const event = parseWebhook({ + header: req.get("x-urlbox-signature"), + rawBody: req.body, // string or Buffer + secret: process.env.URLBOX_WEBHOOK_SECRET!, // or [newSecret, oldSecret] + }); + if (event.event === "render.succeeded") { + console.log(event.renderId, event.result.renderUrl); + } else { + console.log(event.renderId, event.error.message); + } + res.sendStatus(200); + } catch (err) { + if (err instanceof UrlboxWebhookSignatureError) return res.sendStatus(400); + throw err; + } + } +); +``` + +`verifyWebhookSignature` takes the same arguments and returns a boolean. Signatures older than `toleranceSeconds` (default 300) are rejected; pass `0` to turn the check off. + +## Render links + +The default export generates render links: URLs you can put straight into an `` tag or link to. + +### Example ```js import Urlbox from "urlbox"; diff --git a/package.json b/package.json index 3d19e97..47ea26c 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,30 @@ { "name": "urlbox", - "version": "2.2.0", + "version": "2.3.0", "description": "Generate website screenshots, PDFs and other renders from URLs or HTML using the urlbox website screenshot API", "scripts": { "test": "vitest --run", "test:watch": "vitest", "build": "rm -fr dist/* && tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && ./fixup", - "prepublishOnly": "npm run build && npm run test" + "prepublishOnly": "npm run build && npm run test", + "generate:options": "node --experimental-strip-types --no-warnings scripts/generate-render-options.ts", + "typecheck": "tsc -p tsconfig.test.json" }, - "types": "dist/mjs/index.d.ts", + "types": "dist/cjs/index.d.ts", "main": "dist/cjs/index.js", "module": "dist/mjs/index.js", "exports": { ".": { - "import": "./dist/mjs/index.js", - "require": "./dist/cjs/index.js" - } + "import": { + "types": "./dist/mjs/index.d.ts", + "default": "./dist/mjs/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.ts", + "default": "./dist/cjs/index.js" + } + }, + "./package.json": "./package.json" }, "repository": { "type": "git", diff --git a/scripts/generate-render-options.ts b/scripts/generate-render-options.ts new file mode 100644 index 0000000..30bfff8 --- /dev/null +++ b/scripts/generate-render-options.ts @@ -0,0 +1,236 @@ +/** + * Regenerates src/render-options.generated.ts from the canonical Zod render + * options schema in the Urlbox API monorepo. + * + * Usage (Node 22+): + * node --experimental-strip-types scripts/generate-render-options.ts \ + * [path to apps/docs/src/lib/options.ts] + * + * The docs catalogue is optional; when given, its descriptions and defaults + * become JSDoc on each field. + */ +import { writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +type Def = { + type: string; + [key: string]: unknown; +}; +type Schema = { + _zod: { def: Def }; + safeParse: (value: unknown) => { success: boolean; data?: unknown }; + description?: string; +}; +type DocOption = { name: string; desc?: string; defaultVal?: unknown }; + +const [distArg, docsArg] = process.argv.slice(2); +if (!distArg) { + console.error( + "usage: generate-render-options.ts [docs options.ts]", + ); + process.exit(1); +} + +const here = path.dirname(fileURLToPath(import.meta.url)); +const outPath = path.resolve(here, "../src/render-options.generated.ts"); +const SCHEMA_EXPORT = "tightRenderOptionsSchemaNoRequiredOptions"; +const REQUIRED_ONE_OF = new Set(["url", "html"]); + +const typesModule = await import( + pathToFileURL(path.resolve(distArg, "render/render.types.js")).href +); +const schema = typesModule[SCHEMA_EXPORT] as Schema & { + shape: Record; +}; +if (!schema?.shape) { + throw new Error(`${SCHEMA_EXPORT} not found in ${distArg}`); +} + +const docs = new Map(); +if (docsArg) { + const docsModule = await import(pathToFileURL(path.resolve(docsArg)).href); + for (const group of docsModule.options as { options: DocOption[] }[]) { + for (const option of group.options) { + if (!docs.has(option.name)) docs.set(option.name, option); + } + } +} + +const def = (s: Schema): Def => s._zod.def; +const isType = (s: Schema, type: string) => def(s).type === type; +const unionOptions = (s: Schema): Schema[] => + isType(s, "union") ? (def(s).options as Schema[]) : []; +const unionOf = (s: Schema, types: string[]) => { + const options = unionOptions(s).map((o) => def(o).type); + return ( + options.length === types.length && types.every((t) => options.includes(t)) + ); +}; +const quote = (value: unknown) => JSON.stringify(value); + +/** Coercing union -> transform: decide what the transform produces by running it. */ +const coercedType = (pipe: Schema, input: Schema): string | undefined => { + if (unionOf(input, ["string", "boolean", "number"])) return "boolean"; + if (unionOf(input, ["string", "number"])) { + const transform = def(def(pipe).out as Schema).transform as ( + value: unknown, + ctx: unknown, + ) => unknown; + const ctx = { issues: [], addIssue: () => {} }; + const numeric = typeof transform("7", ctx); + const text = typeof transform("top", ctx); + if (numeric === "number" && text === "string") return "string | number"; + return numeric === "number" ? "number" : "string"; + } + const strings = unionOptions(input); + if ( + strings.length === 2 && + strings.some((o) => isType(o, "string")) && + strings.some((o) => isType(o, "array")) + ) { + return "string | string[]"; + } + return undefined; +}; + +const objectType = (shape: Record, indent: string): string => { + const inner = `${indent} `; + const fields = Object.entries(shape).map( + ([key, value]) => + `${inner}${/^[A-Za-z_$][\w$]*$/.test(key) ? key : quote(key)}${isOptional(value) ? "?" : ""}: ${tsType(value, inner)};`, + ); + return fields.length + ? `{\n${fields.join("\n")}\n${indent}}` + : "Record"; +}; + +const isOptional = (s: Schema): boolean => { + const t = def(s).type; + if (t === "optional" || t === "default" || t === "prefault") return true; + if (t === "nullable" || t === "readonly" || t === "catch") { + return isOptional(def(s).innerType as Schema); + } + return false; +}; + +const wrap = (type: string) => (/[|&]/.test(type) ? `(${type})` : type); + +function tsType(s: Schema, indent = " "): string { + const d = def(s); + switch (d.type) { + case "optional": + case "default": + case "prefault": + case "catch": + case "readonly": + case "nonoptional": + return tsType(d.innerType as Schema, indent); + case "nullable": + return `${tsType(d.innerType as Schema, indent)} | null`; + case "pipe": { + const input = d.in as Schema; + const output = d.out as Schema; + // z.preprocess(fn, X): the caller sends what X accepts. + if (isType(input, "transform")) return tsType(output, indent); + if (isType(output, "transform")) { + return coercedType(s, input) ?? tsType(input, indent); + } + return tsType(input, indent); + } + case "union": { + const parts = [ + ...new Set((d.options as Schema[]).map((o) => tsType(o, indent))), + ]; + if (parts.includes("unknown")) return "unknown"; + // Keep literal autocompletion when an open string is also accepted. + const hasLiterals = parts.some((p) => p.startsWith('"')); + return parts + .map((p) => (p === "string" && hasLiterals ? "(string & {})" : p)) + .join(" | "); + } + case "enum": + return [...new Set(Object.values(d.entries as object))] + .map(quote) + .join(" | "); + case "literal": + return (d.values as unknown[]).map(quote).join(" | "); + case "array": + return `${wrap(tsType(d.element as Schema, indent))}[]`; + case "object": + return objectType(d.shape as Record, indent); + case "record": + return `Record`; + case "string": + return "string"; + case "number": + case "int": + case "bigint": + return "number"; + case "boolean": + return "boolean"; + case "null": + return "null"; + default: + return "unknown"; + } +} + +const jsdoc = (key: string, s: Schema): string => { + const doc = docs.get(key); + const lines: string[] = []; + const description = doc?.desc ?? s.description; + if (description) { + lines.push( + ...description + .trim() + .split("\n") + .map((l) => l.trimEnd()), + ); + } + const defaultVal = doc?.defaultVal; + if ( + typeof defaultVal === "string" || + typeof defaultVal === "number" || + typeof defaultVal === "boolean" + ) { + lines.push(`@default ${quote(defaultVal)}`); + } + if (!lines.length) return ""; + const body = lines + .map((l) => ` *${l ? ` ${l.replaceAll("*/", "*\\/")}` : ""}`) + .join("\n"); + return ` /**\n${body}\n */\n`; +}; + +const keys = Object.keys(schema.shape).sort(); +const fields = keys + .filter((key) => !REQUIRED_ONE_OF.has(key)) + .map((key) => { + const field = schema.shape[key]; + return `${jsdoc(key, field)} ${key}?: ${tsType(field)};`; + }); + +const source = `// AUTO-GENERATED by scripts/generate-render-options.ts — do not edit by hand. +// Source: ${SCHEMA_EXPORT} (${keys.length} options) in the Urlbox API. + +/** Every render option except \`url\` / \`html\`, all optional. */ +export interface GeneratedRenderOptionFields { +${fields.join("\n")} +} + +export interface UrlInput { +${jsdoc("url", schema.shape.url)} url: string; + html?: never; +} + +export interface HtmlInput { +${jsdoc("html", schema.shape.html)} html: string; + url?: never; +} +`; + +writeFileSync(outPath, source); +console.log( + `wrote ${path.relative(process.cwd(), outPath)} (${keys.length} options)`, +); diff --git a/src/CHANGELOG.md b/src/CHANGELOG.md index ef08b28..5d50da2 100644 --- a/src/CHANGELOG.md +++ b/src/CHANGELOG.md @@ -1,3 +1,18 @@ +# v2.3.0 + +## Features + +- Add the `Urlbox` API client: `render`, `renderAsync`, `getRender`, `waitForRender` and `generateRenderLink` +- Add `verifyWebhookSignature` and `parseWebhook` for `X-Urlbox-Signature` webhooks, including secret rotation +- Add `UrlboxError`, `UrlboxRenderFailedError` and `UrlboxWebhookSignatureError` +- Add response and webhook types (`RenderResult`, `RenderStatus`, `WebhookEvent`, ...) + +## Improvements + +- `RenderOptions` is now generated from the API's option schema (387 options, with descriptions). Options from earlier releases that the API no longer lists are kept as deprecated fields +- CommonJS consumers now get type declarations, and `exports` lists `types` for both `import` and `require` +- Test files are no longer included in the published package + # v2.2.0 ## Improvements diff --git a/src/client.test.ts b/src/client.test.ts new file mode 100644 index 0000000..eaaef0f --- /dev/null +++ b/src/client.test.ts @@ -0,0 +1,274 @@ +import { describe, expect, test, vi } from "vitest"; +import urlboxFactory, { + Urlbox, + UrlboxError, + UrlboxRenderFailedError, +} from "./index.js"; +import type { RenderStatus } from "./index.js"; + +type Call = { url: string; init: RequestInit }; + +const json = ( + status: number, + body: unknown, + headers: Record = {}, +) => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json", ...headers }, + }); + +const mockFetch = (...responses: Response[]) => { + const calls: Call[] = []; + const fetch = vi.fn( + async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(url), init: init ?? {} }); + const next = responses.shift(); + if (!next) throw new Error("no more mock responses"); + return next; + }, + ); + return { fetch: fetch as unknown as typeof globalThis.fetch, calls }; +}; + +const client = (fetch: typeof globalThis.fetch, extra: object = {}) => + new Urlbox({ apiKey: "pubkey", secret: "sekret", fetch, ...extra }); + +const succeeded = { + renderId: "r1", + status: "succeeded", + renderUrl: "https://renders.urlbox.com/r1.png", + size: 2048, +}; + +describe("Urlbox.render", () => { + test("posts options to /v1/render/sync with bearer auth", async () => { + const body = { + renderUrl: "https://renders.urlbox.com/x.png", + size: 100, + renderTime: 900, + }; + const { fetch, calls } = mockFetch(json(200, body)); + const result = await client(fetch).render({ + url: "https://example.com", + full_page: true, + }); + + expect(result).toEqual(body); + expect(calls[0].url).toBe("https://api.urlbox.com/v1/render/sync"); + expect(calls[0].init.method).toBe("POST"); + const headers = calls[0].init.headers as Record; + expect(headers.Authorization).toBe("Bearer sekret"); + expect(headers["Content-Type"]).toBe("application/json"); + expect(JSON.parse(calls[0].init.body as string)).toEqual({ + url: "https://example.com", + full_page: true, + }); + }); + + test("honours a custom baseUrl", async () => { + const { fetch, calls } = mockFetch(json(200, { renderUrl: "x", size: 1 })); + await client(fetch, { baseUrl: "https://api.example.test/" }).render({ + url: "a.com", + }); + expect(calls[0].url).toBe("https://api.example.test/v1/render/sync"); + }); + + test("maps a 4xx to UrlboxError with status, message, requestId and rate limit", async () => { + const { fetch } = mockFetch( + json( + 429, + { + error: { message: "Rate limit exceeded", code: "RateLimitExceeded" }, + requestId: "body-id", + }, + { + "x-urlbox-error-message": "Rate limit exceeded", + "x-urlbox-request-id": "ps_123", + "x-ratelimit-limit": "60", + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": "1761048000", + "retry-after": "12", + }, + ), + ); + const error = await client(fetch) + .render({ url: "a.com" }) + .catch((e) => e); + expect(error).toBeInstanceOf(UrlboxError); + expect(error.status).toBe(429); + expect(error.message).toBe("Rate limit exceeded"); + expect(error.code).toBe("RateLimitExceeded"); + expect(error.requestId).toBe("ps_123"); + expect(error.rateLimit).toEqual({ + limit: 60, + remaining: 0, + reset: 1761048000, + retryAfter: 12, + }); + expect(error.body).toEqual({ + error: { message: "Rate limit exceeded", code: "RateLimitExceeded" }, + requestId: "body-id", + }); + }); + + test("falls back to the body message and request id without headers", async () => { + const { fetch } = mockFetch( + json(400, { + error: { message: "Invalid options", code: "InvalidOptions" }, + requestId: "ps_9", + }), + ); + const error = await client(fetch) + .render({ url: "a.com" }) + .catch((e) => e); + expect(error.message).toBe("Invalid options"); + expect(error.requestId).toBe("ps_9"); + expect(error.rateLimit).toBeUndefined(); + }); + + test("times out", async () => { + const fetch = ((_url: string, init: RequestInit) => + new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => + reject(new Error("aborted")), + ); + })) as unknown as typeof globalThis.fetch; + const error = await client(fetch, { timeoutMs: 10 }) + .render({ url: "a.com" }) + .catch((e) => e); + expect(error).toBeInstanceOf(UrlboxError); + expect(error.status).toBe(0); + expect(error.code).toBe("TIMEOUT"); + }); +}); + +describe("Urlbox.renderAsync", () => { + test("returns the created render", async () => { + const created = { + status: "created", + renderId: "r1", + statusUrl: "https://api.urlbox.com/v1/render/r1", + }; + const { fetch, calls } = mockFetch(json(201, created)); + expect(await client(fetch).renderAsync({ url: "a.com" })).toEqual(created); + expect(calls[0].url).toBe("https://api.urlbox.com/v1/render/async"); + }); +}); + +describe("Urlbox.getRender", () => { + const cases: RenderStatus[] = [ + { + renderId: "r1", + status: "created", + statusUrl: "https://api.urlbox.com/v1/render/r1", + }, + { + renderId: "r1", + status: "retrying", + statusUrl: "https://api.urlbox.com/v1/render/r1", + stalled: true, + }, + { ...succeeded, status: "succeeded" }, + { + renderId: "r1", + status: "failed", + reason: "Navigation timeout", + statusCode: 504, + }, + ]; + + for (const body of cases) { + test(`returns status ${body.status}`, async () => { + const { fetch, calls } = mockFetch(json(200, body)); + expect(await client(fetch).getRender("r1")).toEqual(body); + expect(calls[0].url).toBe("https://api.urlbox.com/v1/render/r1"); + expect(calls[0].init.method).toBe("GET"); + }); + } + + test("returns not-found from a 404 instead of throwing", async () => { + const { fetch } = mockFetch( + json(404, { renderId: "r1", status: "not-found" }), + ); + expect(await client(fetch).getRender("r1")).toEqual({ + renderId: "r1", + status: "not-found", + }); + }); + + test("throws on other errors", async () => { + const { fetch } = mockFetch(new Response("Unauthorized", { status: 401 })); + const error = await client(fetch) + .getRender("r1") + .catch((e) => e); + expect(error).toBeInstanceOf(UrlboxError); + expect(error.status).toBe(401); + expect(error.message).toBe("Unauthorized"); + }); +}); + +describe("Urlbox.waitForRender", () => { + const pending = { + renderId: "r1", + status: "created", + statusUrl: "https://api.urlbox.com/v1/render/r1", + }; + + test("polls until the render succeeds", async () => { + const { fetch, calls } = mockFetch( + json(200, pending), + json(200, pending), + json(200, succeeded), + ); + const result = await client(fetch).waitForRender("r1", { intervalMs: 1 }); + expect(result).toEqual(succeeded); + expect(calls).toHaveLength(3); + }); + + test("throws UrlboxRenderFailedError when the render fails", async () => { + const failed = { + renderId: "r1", + status: "failed", + reason: "Page returned 404", + }; + const { fetch } = mockFetch(json(200, pending), json(200, failed)); + const error = await client(fetch) + .waitForRender("r1", { intervalMs: 1 }) + .catch((e) => e); + expect(error).toBeInstanceOf(UrlboxRenderFailedError); + expect(error).toBeInstanceOf(UrlboxError); + expect(error.message).toBe("Page returned 404"); + expect(error.body).toEqual(failed); + }); + + test("throws when the wait times out", async () => { + const { fetch } = mockFetch( + ...Array.from({ length: 50 }, () => json(200, pending)), + ); + const error = await client(fetch) + .waitForRender("r1", { intervalMs: 5, timeoutMs: 20 }) + .catch((e) => e); + expect(error).toBeInstanceOf(UrlboxError); + expect(error).not.toBeInstanceOf(UrlboxRenderFailedError); + expect(error.code).toBe("WAIT_TIMEOUT"); + }); +}); + +describe("Urlbox.generateRenderLink", () => { + test("matches the legacy link generator", () => { + const options = { url: "bbc.co.uk", width: 1024, full_page: true }; + const legacy = urlboxFactory( + "pubkey", + "sekret", + "https://api.urlbox.com/v1/", + ); + const urlbox = new Urlbox({ apiKey: "pubkey", secret: "sekret" }); + expect(urlbox.generateRenderLink(options)).toBe( + legacy.generateRenderLink(options), + ); + expect(urlbox.generateRenderLink(options, { insecure: true })).toBe( + legacy.generateInsecureRenderLink(options), + ); + }); +}); diff --git a/src/client.ts b/src/client.ts new file mode 100644 index 0000000..fb1aba0 --- /dev/null +++ b/src/client.ts @@ -0,0 +1,257 @@ +import { UrlboxError, UrlboxRenderFailedError } from "./errors.js"; +import type { RateLimitInfo } from "./errors.js"; +import { buildRenderLink } from "./render-link.js"; +import type { + AsyncRenderCreated, + RenderResult, + RenderStatus, + RenderSucceeded, +} from "./responses.js"; +import type { RenderOptions } from "./types.js"; + +const DEFAULT_BASE_URL = "https://api.urlbox.com"; +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_POLL_INTERVAL_MS = 2_000; +const DEFAULT_WAIT_TIMEOUT_MS = 300_000; + +export interface UrlboxOptions { + /** Your project's publishable API key. */ + apiKey: string; + /** Your project's secret key. */ + secret: string; + /** Default `https://api.urlbox.com`. */ + baseUrl?: string; + /** A `fetch` implementation. Defaults to the global `fetch`. */ + fetch?: typeof fetch; + /** Per-request timeout. Default 120000. */ + timeoutMs?: number; +} + +export interface WaitForRenderOptions { + /** Delay between polls. Default 2000. */ + intervalMs?: number; + /** Give up after this long. Default 300000. */ + timeoutMs?: number; +} + +const numberHeader = (headers: Headers, name: string) => { + const value = headers.get(name); + if (value === null || value.trim() === "") return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +}; + +const rateLimitFrom = (headers: Headers): RateLimitInfo | undefined => { + const info: RateLimitInfo = { + limit: numberHeader(headers, "x-ratelimit-limit"), + remaining: numberHeader(headers, "x-ratelimit-remaining"), + reset: numberHeader(headers, "x-ratelimit-reset"), + retryAfter: numberHeader(headers, "retry-after"), + }; + return Object.values(info).some((v) => v !== undefined) ? info : undefined; +}; + +const readBody = async (response: Response): Promise => { + const text = await response.text(); + if (!text) return undefined; + try { + return JSON.parse(text); + } catch { + return text; + } +}; + +const errorFields = ( + body: unknown, +): { message?: string; code?: string; requestId?: string } => { + if (typeof body !== "object" || body === null) { + return typeof body === "string" && body ? { message: body } : {}; + } + const record = body as Record; + const error = + typeof record.error === "object" && record.error !== null + ? (record.error as Record) + : record; + const asString = (v: unknown) => (typeof v === "string" ? v : undefined); + return { + message: asString(error.message) ?? asString(record.error), + code: asString(error.code), + requestId: asString(record.requestId), + }; +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Client for the Urlbox render API. + * + * ```ts + * const urlbox = new Urlbox({ apiKey, secret }); + * const { renderUrl } = await urlbox.render({ url: "https://example.com" }); + * ``` + */ +export class Urlbox { + private readonly apiKey: string; + private readonly secret: string; + private readonly baseUrl: string; + private readonly fetchImpl: typeof fetch; + private readonly timeoutMs: number; + + constructor(opts: UrlboxOptions) { + if (!opts?.apiKey) throw new Error("apiKey is required"); + if (!opts.secret) throw new Error("secret is required"); + const fetchImpl = opts.fetch ?? globalThis.fetch; + if (!fetchImpl) { + throw new Error("No fetch implementation found; pass one as `fetch`"); + } + this.apiKey = opts.apiKey; + this.secret = opts.secret; + this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + this.fetchImpl = fetchImpl; + this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + } + + /** Renders synchronously and returns the JSON result (`POST /v1/render/sync`). */ + render(options: RenderOptions): Promise { + return this.request("POST", "/v1/render/sync", options); + } + + /** Queues a render (`POST /v1/render/async`). Use `getRender` or `waitForRender` for the result. */ + renderAsync(options: RenderOptions): Promise { + return this.request( + "POST", + "/v1/render/async", + options, + ); + } + + /** Fetches a render's status (`GET /v1/render/:renderId`). */ + getRender(renderId: string): Promise { + return this.request( + "GET", + `/v1/render/${encodeURIComponent(renderId)}`, + undefined, + (status, body) => + status === 404 && + typeof body === "object" && + body !== null && + (body as { status?: unknown }).status === "not-found", + ); + } + + /** + * Polls `getRender` until the render succeeds. Throws `UrlboxRenderFailedError` + * if it fails, and `UrlboxError` if it is not found or the wait times out. + */ + async waitForRender( + renderId: string, + opts: WaitForRenderOptions = {}, + ): Promise { + const intervalMs = opts.intervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const deadline = Date.now() + (opts.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS); + for (;;) { + const status = await this.getRender(renderId); + switch (status.status) { + case "succeeded": + return status; + case "failed": + throw new UrlboxRenderFailedError(status.reason ?? "Render failed", { + status: 200, + code: "RENDER_FAILED", + body: status, + }); + case "not-found": + throw new UrlboxError(`Render ${renderId} not found`, { + status: 404, + code: "RENDER_NOT_FOUND", + body: status, + }); + } + if (Date.now() + intervalMs > deadline) { + throw new UrlboxError(`Timed out waiting for render ${renderId}`, { + status: 0, + code: "WAIT_TIMEOUT", + body: status, + }); + } + await sleep(intervalMs); + } + } + + /** + * Builds a signed render link for these options (unsigned with `insecure`), + * using the same algorithm as the default export. + */ + generateRenderLink( + options: RenderOptions, + opts: { insecure?: boolean } = {}, + ): string { + return buildRenderLink( + `${this.baseUrl}/v1/`, + this.apiKey, + options, + opts.insecure ? undefined : this.secret, + ); + } + + private async request( + method: "GET" | "POST", + path: string, + body?: unknown, + acceptError?: (status: number, body: unknown) => boolean, + ): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), this.timeoutMs); + let response: Response; + let parsed: unknown; + try { + response = await this.fetchImpl(`${this.baseUrl}${path}`, { + method, + headers: { + Accept: "application/json", + Authorization: `Bearer ${this.secret}`, + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: controller.signal, + }); + parsed = await readBody(response); + } catch (error) { + const timedOut = controller.signal.aborted; + throw new UrlboxError( + timedOut + ? `Request timed out after ${this.timeoutMs}ms` + : `Request failed: ${error instanceof Error ? error.message : String(error)}`, + { status: 0, code: timedOut ? "TIMEOUT" : "NETWORK_ERROR" }, + ); + } finally { + clearTimeout(timer); + } + + if (response.ok || acceptError?.(response.status, parsed)) { + if (typeof parsed !== "object" || parsed === null) { + throw new UrlboxError("Expected a JSON response from the Urlbox API", { + status: response.status, + requestId: response.headers.get("x-urlbox-request-id") ?? undefined, + body: parsed, + }); + } + return parsed as T; + } + + const fields = errorFields(parsed); + throw new UrlboxError( + response.headers.get("x-urlbox-error-message") ?? + fields.message ?? + `Urlbox API responded with HTTP ${response.status}`, + { + status: response.status, + requestId: + response.headers.get("x-urlbox-request-id") ?? fields.requestId, + code: fields.code, + body: parsed, + rateLimit: rateLimitFrom(response.headers), + }, + ); + } +} diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..5341d54 --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,58 @@ +export interface RateLimitInfo { + /** `x-ratelimit-limit`: requests allowed in the current window. */ + limit?: number; + /** `x-ratelimit-remaining`: requests left in the current window. */ + remaining?: number; + /** `x-ratelimit-reset`: unix time (seconds) when the window resets. */ + reset?: number; + /** `retry-after`: seconds to wait before retrying. */ + retryAfter?: number; +} + +export interface UrlboxErrorDetails { + status: number; + requestId?: string; + code?: string; + body?: unknown; + rateLimit?: RateLimitInfo; +} + +/** + * An error returned by the Urlbox API. `status` is the HTTP status, or 0 when + * no response was received (network failure or timeout). + */ +export class UrlboxError extends Error { + status: number; + requestId?: string; + code?: string; + body?: unknown; + rateLimit?: RateLimitInfo; + + constructor(message: string, details: UrlboxErrorDetails) { + super(message); + this.name = "UrlboxError"; + this.status = details.status; + this.requestId = details.requestId; + this.code = details.code; + this.body = details.body; + this.rateLimit = details.rateLimit; + Object.setPrototypeOf(this, new.target.prototype); + } +} + +/** Thrown by `waitForRender` when the render finished with status `failed`. */ +export class UrlboxRenderFailedError extends UrlboxError { + constructor(message: string, details: UrlboxErrorDetails) { + super(message, details); + this.name = "UrlboxRenderFailedError"; + } +} + +/** Thrown by `parseWebhook` when the `X-Urlbox-Signature` header does not verify. */ +export class UrlboxWebhookSignatureError extends Error { + constructor(message = "Invalid Urlbox webhook signature") { + super(message); + this.name = "UrlboxWebhookSignatureError"; + Object.setPrototypeOf(this, new.target.prototype); + } +} diff --git a/src/index.ts b/src/index.ts index f197972..b482ebc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,66 +1,53 @@ "use strict"; -import qs from "qs"; -import { createHmac } from "node:crypto"; -import { RenderOptions } from "./types"; +import { buildRenderLink } from "./render-link.js"; +import type { RenderOptions } from "./types.js"; const DEFAULT_PREFIX = "https://api.urlbox.io/v1/"; -const DEFAULT_OPTIONS = { - format: "png", -}; export default ( key: string, secret?: string, - prefix: string = DEFAULT_PREFIX + prefix: string = DEFAULT_PREFIX, ) => { return { generateRenderLink: (options: RenderOptions) => { if (!secret) { throw new Error("your secret is required"); } - options = validateOptions(options); - const query = toQueryString(options); - const token = generateToken(query, secret); - return `${prefix}${key}/${token}/${options.format || "png"}?${query}`; - }, - generateInsecureRenderLink: (options: RenderOptions) => { - options = validateOptions(options); - const query = toQueryString(options); - return `${prefix}${key}/${options.format || "png"}?${query}`; + return buildRenderLink(prefix, key, options, secret); }, + generateInsecureRenderLink: (options: RenderOptions) => + buildRenderLink(prefix, key, options), }; }; -const generateToken = (queryString: string, secret: string): string => - createHmac("sha256", secret).update(queryString).digest("hex"); - -const toQueryString = (options: RenderOptions) => { - const filter = (key: string, value: any) => { - if (key === "format" || key === "token" || !value) { - return; - } - return value; - }; - - return qs.stringify(options, { - filter, - arrayFormat: "repeat", - }); -}; - -const validateOptions = (options: RenderOptions) => { - if (!options) { - throw new Error("no options object passed"); - } - if (options.url && typeof options.url !== "string") { - throw new Error( - "url should be of type string (something like www.google.com)" - ); - } - if (!options.url && !options.html) { - throw new Error("url or html option is required"); - } - return Object.assign({}, DEFAULT_OPTIONS, options); -}; +export type { RenderOptions }; -export { RenderOptions }; +export { Urlbox } from "./client.js"; +export type { UrlboxOptions, WaitForRenderOptions } from "./client.js"; +export { verifyWebhookSignature, parseWebhook } from "./webhooks.js"; +export type { VerifyWebhookArgs } from "./webhooks.js"; +export { + UrlboxError, + UrlboxRenderFailedError, + UrlboxWebhookSignatureError, +} from "./errors.js"; +export type { RateLimitInfo } from "./errors.js"; +export type { + RenderResult, + AsyncRenderCreated, + RenderStatus, + RenderPending, + RenderSucceeded, + RenderFailed, + RenderNotFound, + RenderResponseInfo, + ThumbnailResult, + SliceResult, + BlockDetectionResult, + UnblockSummary, + WebhookEvent, + WebhookMeta, + WebhookRenderSucceeded, + WebhookRenderFailed, +} from "./responses.js"; diff --git a/src/render-link.ts b/src/render-link.ts new file mode 100644 index 0000000..010f24a --- /dev/null +++ b/src/render-link.ts @@ -0,0 +1,58 @@ +import qs from "qs"; +import { createHmac } from "node:crypto"; +import type { RenderOptions } from "./types.js"; + +const DEFAULT_OPTIONS = { + format: "png", +}; + +export const validateOptions = (options: RenderOptions) => { + if (!options) { + throw new Error("no options object passed"); + } + if (options.url && typeof options.url !== "string") { + throw new Error( + "url should be of type string (something like www.google.com)", + ); + } + if (!options.url && !options.html) { + throw new Error("url or html option is required"); + } + return Object.assign({}, DEFAULT_OPTIONS, options); +}; + +const toQueryString = (options: RenderOptions) => { + const filter = (key: string, value: any) => { + if (key === "format" || key === "token" || !value) { + return; + } + return value; + }; + + return qs.stringify(options, { + filter, + arrayFormat: "repeat", + }); +}; + +const generateToken = (queryString: string, secret: string): string => + createHmac("sha256", secret).update(queryString).digest("hex"); + +/** + * Builds a render link: `${prefix}${key}/${token}/${format}?${query}`, or + * without the token when no secret is given. + */ +export const buildRenderLink = ( + prefix: string, + key: string, + options: RenderOptions, + secret?: string, +): string => { + const validated = validateOptions(options); + const query = toQueryString(validated); + const format = validated.format || "png"; + if (secret === undefined) { + return `${prefix}${key}/${format}?${query}`; + } + return `${prefix}${key}/${generateToken(query, secret)}/${format}?${query}`; +}; diff --git a/src/render-options.generated.ts b/src/render-options.generated.ts new file mode 100644 index 0000000..b943fe1 --- /dev/null +++ b/src/render-options.generated.ts @@ -0,0 +1,1444 @@ +// AUTO-GENERATED by scripts/generate-render-options.ts — do not edit by hand. +// Source: tightRenderOptionsSchemaNoRequiredOptions (387 options) in the Urlbox API. + +/** Every render option except `url` / `html`, all optional. */ +export interface GeneratedRenderOptionFields { + accept_cookies?: boolean; + /** + * Sets an `Accept-Language` header on requests to the target URL + * @default "en-US" + */ + accept_lang?: string; + /** + * Sets the accurate of the Geolocation API in metres. + */ + accuracy?: number; + add_cors?: boolean; + allow_coin?: boolean; + /** + * By default, when Urlbox detects an infinite scrolling page, it does not attempt to continue scrolling to the bottom, as this could result in infinite scrolling! If you want to override this behaviour, pass `true` for this option. + * @default false + */ + allow_infinite?: boolean; + /** + * Sets an `Authorization` header on requests to the target URL. + * Can be used to pass an auth token through to the site in order to 'login' before rendering. + */ + authorization?: string; + auto_crop?: boolean; + auto_crop_bg?: string; + auto_crop_threshold?: number; + /** + * Azure storage account name for the destination container. Overrides the project's Azure configuration; prefer the project setting. + */ + azure_account_name?: string; + /** + * Azure container name to save the render to. Overrides the project's Azure configuration; prefer the project setting. + */ + azure_container_name?: string; + /** + * Sets the blob path, including subdirectories and the filename, to use when saving the render in your Azure container - the Azure equivalent of [`s3_path`](#s3_path). Defaults to `renders/{year}/{month}/{day}/{renderId}`. + * + * ~> The extension (e.g. .png, .jpg or .pdf) will be provided automatically, and should not be included in `azure_path`. + */ + azure_path?: string; + /** + * Azure SAS token for the destination container. Overrides the project's Azure configuration; prefer configuring it on the project instead of per request. + */ + azure_sas_token?: string; + base_url?: string; + /** + * Specify a hex code or CSS color string to use as the background color + * + * Some websites don't set a body background colour, and will show up as transparent backgrounds with PNG, or black when using JPG. Use this setting to set a background colour. If the website explicitly sets a transparent background on the html or body elements, this setting will be overridden. + */ + bg_color?: string; + blend_mode?: string; + /** + * Blocks requests from popular advertising networks from loading. + */ + block_ads?: boolean; + /** + * Block data URLs such as `data:image/png;base64,...` + */ + block_data_urls?: boolean; + /** + * Block fetch requests from the target URL. + */ + block_fetch?: boolean; + /** + * Blocks font requests + */ + block_fonts?: boolean; + /** + * Prevents iframe and frame content from loading by blocking non-navigation document requests. The main page will load normally, but any embedded frames/iframes will be blocked. + */ + block_frames?: boolean; + /** + * Blocks image requests + */ + block_images?: boolean; + block_injections?: boolean; + block_manifests?: boolean; + /** + * Block video and audio requests + */ + block_medias?: boolean; + block_other?: boolean; + block_pings?: boolean; + /** + * Prevent requests for javascript scripts from loading + */ + block_scripts?: boolean; + /** + * Prevents WebSocket connections from being established, blocking real-time communication features like live chat, notifications, or dynamic updates. + */ + block_sockets?: boolean; + /** + * Prevent stylesheet requests from loading + */ + block_styles?: boolean; + block_url_keywords?: string | string[]; + /** + * Block requests from specific domains from loading. You can use wildcard characters such as `*` to match subdomains. + */ + block_urls?: string | string[]; + /** + * Block XHR requests from the target URL. + */ + block_xhr?: boolean; + browser?: string; + cache_ignore?: string; + /** + * If your custom bucket is fronted by a CDN, you can set the host name here. + */ + cdn_host?: string; + /** + * This creates a hash of the rendered file, timestamp and options providing proof that a render was taken at a given time. Returns a hash, timestamp and the options used to hash. Checkout our [`guide`]( /guides/certify-a-screenshot ) on certifying a render for more information. + */ + certify?: boolean; + /** + * Specifies an element selector to click before generating a screenshot or PDF + * + * Example: `#clickme` would click an element with `id="clickme"`. + * + * Can be used multiple times to simulate multiple sequential click events. + * + * If the selector matches multiple elements, only the first element will be clicked. + */ + click?: string | string[]; + /** + * Similar to the [`hide_cookie_banners`](#hide_cookie_banners) option, but instead of hiding the banners, this option attempts to click on the 'Accept' button, in order to accept cookies. + */ + click_accept?: boolean; + /** + * Specifies an element selector to click before generating a screenshot or PDF + * + * Example: `.clickme` would click all elements with `class="clickme"`. + * + * Can be used multiple times to simulate multiple sequential click events. + * + * If the selector matches multiple elements, all elements will be clicked. + */ + click_all?: string | string[]; + click_all_x?: string | string[]; + click_at?: string; + click_x?: string | string[]; + /** + * Clip the screenshot to the bounding box specified by `x,y,width,height`. + */ + clip?: string; + cloudrun?: boolean; + /** + * Color profile to embed in the output image (for example sRGB). + */ + color_profile?: "default" | "srgb" | "dp3" | "rec2020" | "scrgblinear" | "hdr10" | "colorspingamma24"; + /** + * Continue rendering even if a navigation error occurs while loading the page. + */ + continue_on_nav_error?: boolean; + /** + * Sets a cookie on the request when loading the URL. + * + * Example: To set the cookie with key `Opt-In` to the value `yes`, you would set the value of this option to `Opt-In=yes`. + * + * Cookies can be passed as an array, to allow setting multiple cookies - e.g.`["Opt-In=yes","Session-Id=DMTIzNDU"]`. + * + * To achieve multiple cookies with render links, just set the cookie option multiple times, like `cookie=Opt-In%3Dyes&cookie=Session-Id%3DDMTIzNDU`. + * + * To set a specific domain on a cookie, you can do the following: `OptIn=yes;Domain=.mydomain.com`. + * + * You can set other attributes for the cookie such as `Path`, `HttpOnly` and `SameSite` + */ + cookie?: string | string[]; + cookies?: string | string[]; + crop_width?: number; + /** + * Inject custom CSS into the page + */ + css?: string; + custom_data_variable?: string; + /** + * Emulate dark mode on websites by setting `prefers-color-scheme: dark` + * @default "false" + */ + dark_mode?: boolean; + debug?: boolean; + debug_url?: string; + debug_video?: boolean; + /** + * The amount of time to wait before Urlbox captures a render in milliseconds. + * @default "0" + */ + delay?: number; + /** + * Some pages have full-height backgrounds whose heights are set to 100% of the viewport. This can cause the backgrounds to get stretched when making a full page screenshot. If you are seeing this behaviour in your full page screenshots, pass `true` for this option. + * @default false + */ + detect_full_height?: boolean; + disable_animations?: boolean; + /** + * Turns off javascript on the target URL. + * + * ~> Enabling this option will prevent `full_page=true` and many other options, because having javascript disabled prevents Urlbox from evaluating code inside the page's context. + * @default "false" + */ + disable_js?: boolean; + /** + * Prevents ligatures from being used. Useful when rendering a PDF, and you want to extract text which contains ligatures. + */ + disable_ligatures?: boolean; + disable_web_security?: boolean; + display_p3?: boolean; + /** + * Pass in a filename which sets the content-disposition header on the response. E.g. `download=myfilename.png` This will make the Urlbox link downloadable, and will prompt the user to save the file as `myfilename.png` + */ + download?: string; + /** + * Device pixel ratio — render at a higher pixel density for sharper images. Companion to `retina`. + */ + dpr?: number; + /** + * Sets the version of the urlbox rendering engine to use when rendering the page. This can be useful for testing how a page will render in the latest version of our rendering engine. + */ + engine_version?: "stable" | "latest" | "lts" | "experimental" | (string & {}); + /** + * Fail the render if a CAPTCHA is detected on the page. + */ + fail_if_captcha?: boolean; + /** + * Fail the render if a Cloudflare Turnstile challenge is detected. + */ + fail_if_cf_turnstile?: boolean; + /** + * Fails the request if the elements specified by `selector` or `wait_for` options are not found on the page after waiting for `wait_timeout`. + * @default "false" + */ + fail_if_selector_missing?: boolean; + /** + * Fails the request if the element specified by `wait_to_leave` option is found on the page after waiting for `wait_timeout`. + * @default "false" + */ + fail_if_selector_present?: boolean; + /** + * Pass in a specific HTTP status code (e.g., `"400"`, `"500"`) or an array of status codes (e.g., `["400", "404", "500"]`) as strings. Urlbox will fail the request if the final response status code matches any of the specified codes. Also accepts `"blank"`: fail the request when the screenshot is blank (at least 99% of its rows are one flat colour) instead of returning an empty image. + */ + fail_on?: string | string[]; + fail_on_3xx?: boolean; + /** + * If `fail_on_4xx=true` and the requested URL returns a status code between 400 and 499, Urlbox fails the request with a 400 error, an error `code` of `fail_on_4xx`, and a message naming the status code (for example `Page returned 404 and fail_on_4xx was true`). + * @default "false" + */ + fail_on_4xx?: boolean; + /** + * If `fail_on_5xx=true` and the requested URL returns a status code between 500 and 599, Urlbox fails the request with a 400 error, an error `code` of `fail_on_5xx`, and a message naming the status code (for example `Page returned 500 and fail_on_5xx was true`). + * @default "false" + */ + fail_on_5xx?: boolean; + /** + * Specify status codes to exclude from [`fail_on`](#fail_on), [`fail_on_4xx`](#fail_on_4xx), or [`fail_on_5xx`](#fail_on_5xx) checks. For example, if you want to fail on all 5xx errors except 503, use `fail_on_5xx=true` with `fail_on_except=503`. + */ + fail_on_except?: string | string[]; + fail_on_metadata_error?: boolean; + fail_on_retry_conditions?: boolean; + finalRetry?: boolean; + fix_full_height?: boolean; + flash?: boolean; + /** + * Generate a fresh render on each request, instead of getting a cached version. + * @default "false" + */ + force?: boolean; + force_dpr?: boolean; + /** + * The output format of the resulting render. + */ + format?: "png" | "jpg" | "jpeg" | "pdf" | "avif" | "mp4" | "webm" | "webp" | "html" | "mhtml" | "svg" | "md"; + fragment?: boolean; + frame_selection_mode?: "visible" | "all"; + /** + * Detect fixed/sticky elements (headers, footers) and capture them only once, instead of repeating them across each stitched full-page section. + * @default "true" + */ + freeze_fixed?: boolean; + from_html?: boolean; + full_html?: boolean; + /** + * Specify whether to capture the full scrollable area of the website. + * For PDFs, `full_page` mode will attempt to capture the whole website onto one single page PDF document. + * It's likely you'll want to also hide any cookie banners that crop up during a full page screenshot, so we recommend you use [`click_accept`](#click_accept) and [`hide_cookie_banners`](#hide_cookie_banners) too. + * @default "false" + */ + full_page?: boolean; + /** + * Whether to use scroll and stitch algorithm (the default) to render a full page screenshot, or to use the native full page screenshot algorithm, which is faster, but can be less accurate on some sites. + * @default "stitch" + */ + full_page_mode?: "native" | "stitch"; + full_page_slice_height?: number; + full_page_slice_overlap_height?: number; + full_page_slices?: boolean; + /** + * When full_page=true, specify whether to capture the full width of the website, for example if the site is horizontally scrolling. + * @default "false" + */ + full_width?: boolean; + /** + * Enable GPU acceleration to render 3D scenes and heavy WebGL content. This is a beta feature and requires pre-approval. Please contact support@urlbox.com to enable this feature on your account. + * @default "false" + */ + gpu?: boolean; + hdr10?: boolean; + /** + * Set a header on the request when loading the URL + * + * Example: To set the header with key `X-My-Header` to the value `SomeValue`, you would pass `header=X-My-Header%3DSomeValue`. + * + * This can be set multiple times, to set more than one header - e.g. `header=X-My-Header%3DSomeValue&header=X-My-Other-Header%3DSomeOtherValue`. + * + * As with all options passed via the query string, the header value must be URL encoded - so `X-My-Header=SomeValue` becomes `X-My-Header%3DSomeValue` in order to be interpreted correctly by Urlbox. + */ + header?: string | string[]; + headers?: string | string[]; + headless?: "shell" | boolean; + /** + * The viewport height of the browser, in pixels. + * @default 1024 + */ + height?: number; + height_from?: string; + /** + * Automatically hides cookie banners from most websites, by setting their style to `display: none !important;` + */ + hide_cookie_banners?: boolean; + /** + * A lighter-weight alternative to [`use_stealth`](#use_stealth) for hiding that the browser is automated. It runs Chrome with a set of stealth plugins that patch the fingerprints headless Chrome normally leaks (for example `navigator.webdriver`, missing `chrome` runtime objects, and WebGL vendor strings), so pages are less likely to detect the render as a bot. + * + * [`use_stealth`](#use_stealth) is the stronger option and takes precedence if both are set. Reach for `hide_headless` when a site does light bot-detection that a plain render trips but doesn't need the full stealth browser. + * @default "false" + */ + hide_headless?: boolean; + /** + * Hide specific HTML elements on the page before rendering. This option accepts a comma-delimited string of CSS element selectors that will be hidden by setting their style to `visibility: hidden !important; pointer-events: none !important;`. This preserves the page layout while making elements invisible. + * + * This is particularly useful for: + * - Hiding pop-ups, banners, or cookie notices + * - Removing advertisements or promotional overlays + * - Excluding navigation menus or sidebars + * - Hiding specific content sections + * + * **Selector types supported:** + * - **Element selectors** (e.g., `h1`, `div`, `img`) - Hide all elements of that type + * - **Class selectors** (e.g., `.popup`, `.banner`) - Hide elements with specific CSS classes + * - **ID selectors** (e.g., `#header`, `#sidebar`) - Hide elements with specific IDs + * - **Complex selectors** (e.g., `.nav ul li`, `div.content > p`) - Use any valid CSS selector + * - **Multiple selectors** - Combine multiple selectors with commas + * + * **Tip:** To find the selector for any element, open your browser's DevTools, right-click the element in the Elements tab, and select "Copy > Copy selector". + */ + hide_selector?: string; + hide_selector_x?: string; + /** + * Specify a string to highlight on the page before capturing a screenshot or PDF. To highlight multiple words, separate words with a pipe character e.g. Hello|World + */ + highlight?: string; + /** + * Specify the background color of the highlighted word. + * @default "red" + */ + highlightbg?: string; + /** + * Specify the text color of the highlighted word. + * @default "white" + */ + highlightfg?: string; + /** + * Specifies an element selector to hover over before generating a screenshot or PDF + * + * Example: `#hoverme` would hover over the element with `id="hoverme"` + */ + hover?: string | string[]; + hover_x?: string | string[]; + https_upgrade?: boolean; + ignore_load_errors?: boolean; + /** + * Background colour to use when [img_fit](#img_fit) is `contain`, or [`img_pad`](#img_pad) is used, defaults to black without transparency + */ + img_bg?: string; + /** + * How the screenshot should be resized or cropped to fit the dimensions when using [`thumb_width`](#thumb_width) and/or [`thumb_height`](#thumb_height) options + * @default "cover" + */ + img_fit?: "cover" | "contain" | "fill" | "inside" | "outside"; + /** + * Pad the screenshot, giving it a border. Can either be a single pixel value that gets added to each side, or a comma delimited string of `top,right,bottom,left` pixel values. + */ + img_pad?: string; + /** + * How the image should be positioned when using an [`img_fit`](#img_fit) of `cover` or `contain`. + * @default "center" + */ + img_position?: "top" | "right top" | "right" | "right bottom" | "bottom" | "left bottom" | "left" | "left top" | "center" | "centre" | "north" | "northeast" | "east" | "southeast" | "south" | "southwest" | "west" | "northwest" | "entropy" | "attention"; + /** + * Execute custom JavaScript in the context of the page. The JS gets executed after the page's dom has loaded, but before the screenshot is taken. No need to use `load` etc event handlers to run code, as these events will already have fired by the time this JS gets executed. You can use `await` to wait for promises to resolve. + */ + js?: string; + json?: boolean; + kill_popups?: boolean; + /** + * Sets the latitude used to emulate the Geolocation API. + */ + latitude?: number; + lazyload?: boolean; + lightweight?: boolean; + /** + * AWS access key ID for Amazon Bedrock authentication. + */ + llm_aws_access_key_id?: string; + /** + * AWS region for Amazon Bedrock. Defaults to 'us-east-1'. + */ + llm_aws_region?: string; + /** + * AWS secret access key for Amazon Bedrock authentication. + */ + llm_aws_secret_access_key?: string; + /** + * Optional AWS session token for temporary credentials with Amazon Bedrock. + */ + llm_aws_session_token?: string; + llm_azure_api_protocol?: "openai" | "anthropic"; + /** + * Azure OpenAI API version. Required when using deployment URLs. E.g. '2024-02-15-preview'. + */ + llm_azure_api_version?: string; + /** + * Azure OpenAI resource name. Used to construct the endpoint URL: `https://{resourceName}.openai.azure.com/` + */ + llm_azure_resource_name?: string; + /** + * Use legacy Azure deployment URL format. Useful for compatibility with certain Azure OpenAI models or deployments that require the legacy endpoint format. + */ + llm_azure_use_deployment_urls?: boolean; + /** + * Override the default API endpoint for the LLM provider. Useful for proxies, self-hosted models, or custom endpoints. + */ + llm_base_url?: string; + /** + * Return the full LLM response including usage statistics and metadata, rather than just the text content. + */ + llm_full_response?: boolean; + /** + * Google Cloud region for Vertex AI. Defaults to 'us-central1'. + */ + llm_gcp_location?: string; + /** + * Google Cloud project ID for Google Vertex AI. + */ + llm_gcp_project?: string; + /** + * Google Cloud service account JSON key for Vertex AI authentication. Pass the entire JSON key file contents as a string. + */ + llm_gcp_service_account_json?: string; + /** + * The height of the thumbnail image that is sent to the LLM. Defaults to 512. + */ + llm_height?: number; + /** + * The API access key for the given LLM provider. + */ + llm_key?: string; + /** + * The max number of output tokens the LLM can generate a response with. Defaults to 1000. + */ + llm_max_tokens?: number; + llm_md_budget?: number; + llm_md_mode?: "Compact" | "Extractive" | "Full" | "compact" | "extractive" | "full"; + llm_md_tables?: "Drop" | "Full" | "Summary" | "drop" | "full" | "summary"; + /** + * The LLM model to use for the given provider. E.g. 'gpt-5.1' for OpenAI, 'claude-sonnet-4-5-20250929' for Anthropic. + */ + llm_model?: string; + /** + * You can provide a structured output type and schema to Urlbox, and we will prompt your LLM to give back that response structure. By passing an llm_schema without this option, it will default to responding with an object specified by your JSON Schema. If you choose Array, the response will be an array of your provided JSON Schema. If you choose enum, you can provide an array of strings as your schema, and your LLM will respond only with a value from that enum. + */ + llm_output?: "object" | "array" | "enum"; + /** + * The prompt to give the LLM EG "Analyse this image and its associated HTML, giving me back a summary of what the website contents are and a list of all of the links it has " + */ + llm_prompt?: string; + /** + * The LLM provider to use. + */ + llm_provider?: "anthropic" | "openai" | "google" | "azure" | "mistral" | "cohere" | "amazon-bedrock" | "google-vertex" | "groq" | "xai" | "deepseek" | "perplexity" | "togetherai" | "fireworks" | "cerebras" | "openrouter"; + /** + * This is the JSON schema or string[] provided which we will pass over to your LLM. Your LLM provider will respond with a structured output (if supported by your provider) according to that schema. Please take a look at the various resources on the [JSON Schema](https://json-schema.org/tools?query=&sortBy=name&sortOrder=ascending&groupBy=toolingTypes&licenses=&languages=&drafts=&toolingTypes=&environments=&showObsolete=false&supportsBowtie=false) website for more information on designing and validating a JSON schema. + */ + llm_schema?: string[] | string | Record; + /** + * This can be used to provide more overall context for the AI's response. + */ + llm_system_prompt?: string; + /** + * The temperature (creativity) for the LLM prompt. Defaults to 0 for less creative responses. + */ + llm_temperature?: number; + /** + * The width of the thumbnail image that is sent to the LLM. Defaults to 512. + */ + llm_width?: number; + /** + * Sets the longitude used to emulate the Geolocation API. + */ + longitude?: number; + markdown_options?: { + autolinks?: boolean; + brInTables?: boolean; + bullets?: string; + codeBlockStyle?: "Backticks" | "Indented" | "Tildes" | "backticks" | "indented" | "tildes"; + codeLanguage?: string; + convertAsInline?: boolean; + debug?: boolean; + defaultTitle?: boolean; + encoding?: string; + escapeAscii?: boolean; + escapeAsterisks?: boolean; + escapeMisc?: boolean; + escapeUnderscores?: boolean; + extractMetadata?: boolean; + headingStyle?: "Atx" | "AtxClosed" | "Underlined" | "atx" | "atx_closed" | "atxclosed" | "underlined"; + highlightStyle?: "Bold" | "DoubleEqual" | "Html" | "None" | "bold" | "double_equal" | "doubleequal" | "html" | "none"; + hocrSpatialTables?: boolean; + keepInlineImagesIn?: string | string[]; + listIndentType?: "Spaces" | "Tabs" | "spaces" | "tabs"; + listIndentWidth?: number; + newlineStyle?: "Backslash" | "Spaces" | "backslash" | "spaces"; + outputFormat?: "Djot" | "Markdown" | "Plain" | "djot" | "markdown" | "plain"; + preprocessing?: { + enabled?: boolean; + preset?: "Aggressive" | "Minimal" | "Standard" | "aggressive" | "minimal" | "standard"; + removeForms?: boolean; + removeNavigation?: boolean; + }; + preserveTags?: string | string[]; + skipImages?: boolean; + stripNewlines?: boolean; + stripTags?: string | string[]; + strongEmSymbol?: "*" | "_"; + subSymbol?: string; + supSymbol?: string; + whitespaceMode?: "Normalized" | "Strict" | "normalized" | "strict"; + wrap?: boolean; + wrapWidth?: number; + }; + /** + * Maximum total number of render attempts (the original render plus retries) when using [`retry_on`](#retry_on). Must be between 0 and 5. An alternative to [`max_retries`](#max_retries): `max_attempts` is equivalent to `max_retries + 1`, and takes precedence over it when both are set. + * @default "3" + */ + max_attempts?: number; + /** + * For extremely lengthy websites, it may be preferable to limit the screenshot to a maximum height to prevent Urlbox from spending time scrolling and generating an enormous screenshot. + */ + max_height?: number; + /** + * Maximum number of retry attempts when using [`retry_on`](#retry_on). Must be between 0 and 5. If [`max_attempts`](#max_attempts) is also set, `max_attempts` takes precedence. + * @default "2" + */ + max_retries?: number; + max_scroll_page_time?: number; + /** + * When Urlbox takes a `full_page` screenshot, the maximum height of each image section is set to 4096 pixels. If a sites height is greater than this value, Urlbox will start splitting the screenshot into sections. Sometimes it is worthwhile experimenting with this number. + * @default 4096 + */ + max_section_height?: number; + max_section_width?: number; + /** + * Maximum number of screenshot sections to capture when stitching a full-page screenshot. + */ + max_sections?: number; + /** + * Maximum width of the render, in pixels. + */ + max_width?: number; + max_xsections?: number; + md_autolinks?: boolean; + md_br_tables?: boolean; + md_bullets?: string; + md_code_blocks?: "Backticks" | "Indented" | "Tildes" | "backticks" | "indented" | "tildes"; + md_code_lang?: string; + md_debug?: boolean; + md_default_title?: boolean; + md_em?: "*" | "_"; + md_encoding?: string; + md_escape_ascii?: boolean; + md_escape_asterisks?: boolean; + md_escape_misc?: boolean; + md_escape_underscores?: boolean; + md_extract_metadata?: boolean; + md_heading?: "Atx" | "AtxClosed" | "Underlined" | "atx" | "atx_closed" | "atxclosed" | "closed" | "underlined"; + md_highlight?: "Bold" | "DoubleEqual" | "Html" | "None" | "bold" | "double_equal" | "doubleequal" | "html" | "marker" | "none" | "plain"; + md_hocr_tables?: boolean; + md_inline?: boolean; + md_keep_images_in?: string | string[]; + md_list_indent?: "Spaces" | "Tabs" | "spaces" | "tabs"; + md_list_width?: number; + md_newline?: "Backslash" | "Spaces" | "backslash" | "spaces"; + md_output?: "Djot" | "Markdown" | "Plain" | "djot" | "markdown" | "plain"; + md_pre_forms?: boolean; + md_pre_nav?: boolean; + md_preprocess?: boolean; + md_preserve_tags?: string | string[]; + md_preset?: "Aggressive" | "Minimal" | "Standard" | "aggressive" | "minimal" | "standard"; + md_skip_images?: boolean; + md_strip_newlines?: boolean; + md_strip_tags?: string | string[]; + md_sub?: string; + md_sup?: string; + md_whitespace?: "Normalized" | "Strict" | "normalized" | "strict"; + md_wrap?: boolean; + md_wrap_width?: number; + /** + * By default, when generating a PDF, the `print` CSS media query is used. To generate a PDF using the `screen` CSS, set this option to `screen`. + * + * When generating an image, the `screen` CSS media query is used by default. To generate an image using the `print` CSS, set this option to `print`. + */ + media?: "print" | "screen"; + metadata?: boolean; + metadata_engine?: "Legacy" | "Structured" | "legacy" | "structured"; + /** + * Minimum expected file size in bytes. Used with [`retry_on: "small_size"`](#retry_on) to retry when the screenshot is smaller than expected, which may indicate an error page was captured instead of the intended content. + * @default "1000" + */ + min_size_bytes?: number; + minidelay?: number; + mock?: boolean; + mock_cost?: number; + /** + * By default, urlbox adds the file extension (e.g. `.png`, `.jpg`, `.pdf` etc) to the `s3_path` (or [`azure_path`](#azure_path)). + * + * If `no_suffix=true`, the file extension will NOT be added to the path. + */ + no_suffix?: boolean; + no_upload?: boolean; + /** + * Sets the author metadata field of the PDF document. This is visible in PDF readers under document properties. + */ + pdf_author?: string; + /** + * Automatically remove white space from PDF. Occasionally a PDF will have a lot of trailing white space at the bottom of the page. This option will attempt to automatically crop the PDF to remove this white space. + */ + pdf_auto_crop?: boolean; + /** + * Sets whether to print background images in the PDF + * @default "true" + */ + pdf_background?: boolean; + /** + * Sets the creator metadata field of the PDF document. This typically indicates the application that created the original content. Visible in PDF readers under document properties. + */ + pdf_creator?: string; + pdf_dpi?: number; + pdf_fit_to_page?: boolean; + /** + * Change the default pdf footer that is shown on each page of the pdf when [`pdf_show_footer`](#pdf_show_footer) option is set. + * + * You have the option to show the following variables in the footer (or header) of the pdf: + * + * * current `date` + * * `title` of the page + * * `url` of the page + * * current `pageNumber` + * * the `totalPages` in the pdf document + * + * You can display these variables by creating empty divs or spans, with special css class names relating to the variable you want to show. + * + * For example, if you want to show the `date` followed by the `url`, you could use the following pdf footer template: + * + * `
`. + * + * The pdf footer template you set are inserted as the innerHTML of a parent div which is a flex container, and has `align-items` set to `flex-end`. + * + * There are also some helper classes for aligning the divs or spans. The following classes are available: + * + * * `left` - adds some left padding to the element and sets `flex: none`. + * * `center` - aligns the element and text to the center. + * * `right` - adds some right padding to the element and sets `flex: none`. + * * `text` - sets the text to 8pt. + * * `grow` - sets `flex: auto` on the element, allowing it to grow to fill the available space. + * + * + * The default pdf footer is: + * + * `
/
`. + * + * You can see exactly how the pdf page is constructed by looking at the [chromium pdf template](https://source.chromium.org/chromium/chromium/src/+/main:components/printing/resources/print_header_footer_template_page.html;l=101-105?q=header_footer%20&ss=chromium%2Fchromium%2Fsrc) in the chromium source repository. + */ + pdf_footer?: string; + /** + * Change the default pdf header that is shown on each page of the pdf when [`pdf_show_header`](#pdf_show_header) option is set. + * + * You have the option to show the following variables in the header (or footer) of the pdf: + * + * * current `date` + * * `title` of the page + * * `url` of the page + * * current `pageNumber` + * * the `totalPages` in the pdf document + * + * You can display these variables by creating empty divs or spans, with special css class names relating to the variable you want to show. + * + * For example, if you want to show the `date` followed by the `url`, you could use the following pdf header template: + * + * `
`. + * + * The pdf header template you set are inserted as the innerHTML of a parent div which is a flex container, and has `align-items` set to `flex-start`. + * + * There are also some helper classes for aligning the divs or spans. The following classes are available: + * + * * `left` - adds some left padding to the element and sets `flex: none`. + * * `center` - aligns the element and text to the center. + * * `right` - adds some right padding to the element and sets `flex: none`. + * * `text` - sets the text to 8pt. + * * `grow` - sets `flex: auto` on the element, allowing it to grow to fill the available space. + * + * + * The default pdf header is: + * + * `
`. + * + * You can see exactly how the pdf page is constructed by looking at the [chromium pdf template](https://source.chromium.org/chromium/chromium/src/+/main:components/printing/resources/print_header_footer_template_page.html;l=98-100?q=header_footer%20&ss=chromium%2Fchromium%2Fsrc) in the chromium source repository. + */ + pdf_header?: string; + /** + * Sets the keywords metadata field of the PDF document. Pass a comma-separated list of keywords, e.g. `"pdf_keywords": "screenshot,web,api"`. This is visible in PDF readers under document properties and can help with document organization and search. + */ + pdf_keywords?: string; + /** + * Sets the margin of the PDF document. + * @default "none" + */ + pdf_margin?: "default" | "none" | "minimum"; + /** + * Sets a custom bottom margin on the PDF. + */ + pdf_margin_bottom?: number; + /** + * Set a custom left margin on the PDF. + */ + pdf_margin_left?: number; + /** + * Sets a custom right margin on the PDF. + */ + pdf_margin_right?: number; + /** + * Sets a custom top margin on the PDF. + */ + pdf_margin_top?: number; + /** + * Sets the orientation of the PDF. + * @default "portrait" + */ + pdf_orientation?: "portrait" | "landscape"; + /** + * Include a document outline (bookmarks) in the generated PDF. + */ + pdf_outline?: boolean; + /** + * Sets the PDF page height, in pixels. + */ + pdf_page_height?: number; + /** + * Sets the PDF page range to return. + * + * By default, the page is split into a multi page document and returns all page. Use this option to restrict which pages should be returned. + */ + pdf_page_range?: string; + /** + * Sets the PDF page size. + * + * Setting this option will take precedence over `pdf_page_width` and `pdf_page_height`. + * @default "A4" + */ + pdf_page_size?: "a0" | "a1" | "a2" | "a3" | "a4" | "a5" | "a6" | "legal" | "ledger" | "letter" | "tabloid" | "A0" | "A1" | "A2" | "A3" | "A4" | "A5" | "A6" | "Legal" | "Ledger" | "Letter" | "Tabloid"; + /** + * Sets the PDF page width, in pixels. + */ + pdf_page_width?: number; + /** + * Sets the scale factor of the website content in the PDF. Valid values are numbers between 0.1 and 2. + * @default "1" + */ + pdf_scale?: number; + /** + * Whether to show the default pdf footer on each page of the pdf. The template of the footer can be changed by setting the [`pdf_footer`](#pdf_footer) option. + */ + pdf_show_footer?: boolean; + /** + * Whether to show the default pdf header on each page of the pdf. The template of the header can be changed by setting the [`pdf_header`](#pdf_header) option. + */ + pdf_show_header?: boolean; + /** + * Sets the subject metadata field of the PDF document. This is visible in PDF readers under document properties. + */ + pdf_subject?: string; + /** + * Produce a tagged (accessible) PDF. + */ + pdf_tagged?: boolean; + /** + * Sets the title metadata field of the PDF document. This is visible in PDF readers under document properties. If not set, the page's `` tag will be used as the PDF title. + */ + pdf_title?: string; + /** + * Sets the `navigator.platform` that the browser will report for the request. Useful for getting around certain scripts that detect the platform. + * @default "MacIntel" + */ + platform?: string; + point_of_view?: string; + point_of_view_country?: string; + pov?: string; + pov_country?: string; + /** + * Attempts to press the Escape (ESC) key before capturing the page. Useful for dismissing pop-ups, overlays, or advertising banners that appear on load. + */ + press_escape?: boolean; + /** + * Pass in a proxy server address to make screenshot requests via that server in the format `[address]:[port]`. + * + * If proxy authentication is required, you can use the following format: `[user]:[password]@[address]:[port]`. + */ + proxy?: string; + proxy_hosts?: string | string[]; + proxy_resources?: string | string[]; + proxy_same_domain?: boolean; + /** + * The image quality of the resulting screenshot (JPEG/WebP only) + * @default "80" + */ + quality?: number; + /** + * Make the pdf into a readable document by removing unnecessary elements such as navigation bars, ads, etc. + */ + readable?: boolean; + rec2020?: boolean; + /** + * For the synchronous endpoint, the time in milliseconds before Urlbox returns a 307 redirect for a long-running render, so the connection isn't cut by upstream timeouts. + */ + redirect_after?: number; + reduce_proxy?: boolean; + /** + * Prefer less animations on websites by setting `prefers-reduced-motion: reduced` + */ + reduced_motion?: boolean; + refresh?: boolean; + refresh_after_scroll?: boolean; + region?: string; + reload?: boolean; + reload_after_scroll?: boolean; + render_queue?: string; + request_interception_method?: "puppeteer" | "cdp"; + /** + * For render link requests, setting this option to `json` will change the response type of the Urlbox request to JSON. For the API, the default response type is JSON. + */ + response_type?: "json" | "jsondebug" | "binary" | "base64" | "none"; + /** + * Take a 'retina' or high-definition screenshot, equivalent to setting a device pixel ratio of 2.0 or @2x. Please note that retina screenshots will be double the normal dimensions and will normally take slightly longer to process due to the much bigger image size. + * @default "false" + */ + retina?: boolean; + /** + * Base delay in milliseconds between retry attempts. The delay doubles with each retry (exponential backoff), with each individual wait capped at 30 seconds. Must be between 100 and 60000. + * @default "1000" + */ + retry_delay_ms?: number; + /** + * Automatically retry renders when specific conditions occur. Pass a single condition, a comma-separated string, or an array of conditions. + * + * When a condition matches, Urlbox will retry with exponential backoff (the delay doubles each time, starting at 1 second by default and capped at 30 seconds per wait) up to 2 times by default (3 total attempts including the original render). Use [`max_retries`](#max_retries) or [`max_attempts`](#max_attempts) to change this. Retrying stops early if the request approaches the overall 5 minute render budget. + * + * **HTTP status conditions** (retry based on the page's response): + * + * - **`4xx`** - Retry when the page returns any 4xx status. This range deliberately skips retrying when the failure looks like a problem with the request itself rather than a block (for example a missing selector or an invalid option), since those would fail identically on every attempt. List codes like `403`/`429` explicitly to always retry them. + * - **`5xx`** - Retry when the page returns any 5xx status + * - **`404`**, **`429`**, **`503`**, etc. - Retry on specific status codes + * + * **Engine conditions** (retry when Urlbox encounters an internal error): + * + * - **`timeout`** - Retry when the render times out + * - **`crash`** - Retry when the browser crashes + * + * **Quality conditions:** + * + * - **`small_size`** - Retry when the screenshot is smaller than [`min_size_bytes`](#min_size_bytes) + * - **`blank`** - Retry when the screenshot is blank: at least 99% of its rows are one flat colour. Catches pages whose content never appeared (hidden behind a dismissed popup, or not yet loaded). Not included in `all`. + * + * **Special values:** + * + * - **`all`** - Retry on any of the conditions above + * + * **Examples:** + * + * - `retry_on: "4xx"` - Retry when the page returns any 4xx status + * - `retry_on: ["429", "503"]` - Retry on rate limiting or service unavailable + * - `retry_on: "timeout,crash"` - Retry on engine failures only + * - `retry_on: "5xx,timeout"` - Retry on page 5xx errors or timeouts + * + * **Note:** If a condition matches both `retry_on` and [`fail_on`](#fail_on), the request will be retried first. Use [`fail_on_except`](#fail_on_except) to mark specific status codes as acceptable (won't retry or fail). + */ + retry_on?: string | string[]; + retry_on_nav_error?: boolean; + /** + * Change render options on retry attempts triggered by [`retry_on`](#retry_on). Instead of retrying with the exact same options, each retry re-runs the render with the `retry_with` options merged over the original request, so you can escalate progressively: try a cheap, plain render first, and only add stealth or a proxy if the render fails. + * + * `retry_with` has no effect on its own; it only applies when [`retry_on`](#retry_on) triggers a retry. Because the value is a nested object, use it with JSON POST requests to the render endpoints. + * + * **Single object** - applied to every retry attempt. **Array of objects** - progressive escalation: the first retry uses the first element, the second retry the second, and so on; the last element is reused if there are more retries than elements. + * + * **Merging rules:** each retry's options are the original request options with that attempt's `retry_with` entry merged on top. Setting an option to `null` removes it for that attempt; options you don't mention are left unchanged. Only options that affect how the page is fetched and rendered can be changed on retry (proxy, stealth, waiting, viewport, headers/cookies and similar) - options that change the output itself (like `format` or `full_page`) cannot. + */ + retry_with?: { + proxy?: string | null; + use_proxy?: boolean | null; + urlbox_proxy?: string | null; + reduce_proxy?: boolean | null; + proxy_same_domain?: boolean | null; + proxy_resources?: string | string[] | null; + proxy_hosts?: string | string[] | null; + user_agent?: string | null; + pov?: string | null; + point_of_view?: string | null; + pov_country?: string | null; + point_of_view_country?: string | null; + use_stealth?: boolean | null; + hide_headless?: boolean | null; + block_ads?: boolean | null; + block_urls?: string | string[] | null; + block_url_keywords?: string | string[] | null; + hide_cookie_banners?: boolean | null; + click_accept?: boolean | null; + click?: string | string[] | null; + click_all?: string | string[] | null; + hover?: string | string[] | null; + wait_until?: "networkidle0" | "networkidle2" | "load" | "domcontentloaded" | "smart" | null; + wait_for?: string | null; + delay?: number | null; + width?: number | null; + height?: number | null; + retina?: boolean | null; + viewport_clip?: boolean | null; + js?: string | null; + css?: string | null; + hide_selector?: string | null; + header?: string | string[] | null; + cookie?: string | string[] | null; + authorization?: string | null; + accept_lang?: string | null; + media?: "screen" | "print" | null; + ignore_https_errors?: boolean | null; + disable_web_security?: boolean | null; + timeout?: number | null; + debug?: boolean | null; + headless?: boolean | null; + } | ({ + proxy?: string | null; + use_proxy?: boolean | null; + urlbox_proxy?: string | null; + reduce_proxy?: boolean | null; + proxy_same_domain?: boolean | null; + proxy_resources?: string | string[] | null; + proxy_hosts?: string | string[] | null; + user_agent?: string | null; + pov?: string | null; + point_of_view?: string | null; + pov_country?: string | null; + point_of_view_country?: string | null; + use_stealth?: boolean | null; + hide_headless?: boolean | null; + block_ads?: boolean | null; + block_urls?: string | string[] | null; + block_url_keywords?: string | string[] | null; + hide_cookie_banners?: boolean | null; + click_accept?: boolean | null; + click?: string | string[] | null; + click_all?: string | string[] | null; + hover?: string | string[] | null; + wait_until?: "networkidle0" | "networkidle2" | "load" | "domcontentloaded" | "smart" | null; + wait_for?: string | null; + delay?: number | null; + width?: number | null; + height?: number | null; + retina?: boolean | null; + viewport_clip?: boolean | null; + js?: string | null; + css?: string | null; + hide_selector?: string | null; + header?: string | string[] | null; + cookie?: string | string[] | null; + authorization?: string | null; + accept_lang?: string | null; + media?: "screen" | "print" | null; + ignore_https_errors?: boolean | null; + disable_web_security?: boolean | null; + timeout?: number | null; + debug?: boolean | null; + headless?: boolean | null; + })[]; + revisit?: boolean; + revisit_after_scroll?: boolean; + /** + * Overrides the configured bucket to use when saving the render. + */ + s3_bucket?: string; + /** + * You can change the endpoint URL to use an S3 compatible storage provider e.g. DigitalOcean Spaces, Minio, Wasabi, Cloudflare R2 and more. + */ + s3_endpoint?: string; + s3_force_path_style?: boolean; + /** + * S3 access key ID for the destination bucket. This overrides the storage credentials configured on your project — we recommend configuring storage on the project instead of sending keys per request. + */ + s3_key?: string; + s3_object_lock?: boolean; + s3_part_size?: number; + /** + * Sets the S3 path, including subdirectories and the filename, to use when saving the render in your S3-compatible bucket. + * + * ~> The extension (e.g. .png, .jpg or .pdf) will be provided automatically, and should not be included in `s3_path`. + */ + s3_path?: string; + /** + * Return a time-limited presigned URL to the render in your S3 bucket, instead of the default response. The URL grants temporary access without exposing your credentials. + */ + s3_presigned_url?: string; + /** + * Return a presigned URL for the saved HTML side-render. + */ + s3_presigned_url_html?: string; + /** + * Return a presigned URL for the saved Markdown side-render. + */ + s3_presigned_url_markdown?: string; + /** + * Return a presigned URL for the saved metadata side-render. + */ + s3_presigned_url_metadata?: string; + s3_presigned_url_mhtml?: string; + s3_presigned_url_resource_info?: string; + s3_private_bucket?: boolean; + /** + * Override the configured S3 region when saving the render. + */ + s3_region?: string; + /** + * S3 secret access key for the destination bucket. Overrides the project's storage credentials; prefer configuring storage on the project rather than per request. + */ + s3_secret?: string; + /** + * Sets the s3 storage class. + * @default "standard" + */ + s3_storageclass?: "STANDARD" | "REDUCED_REDUNDANCY" | "STANDARD_IA" | "ONEZONE_IA" | "INTELLIGENT_TIERING" | "GLACIER" | "DEEP_ARCHIVE" | "OUTPOSTS" | "standard" | "reduced_redundancy" | "standard_ia" | "onezone_ia" | "intelligent_tiering" | "glacier" | "deep_archive" | "outposts"; + save_clicks?: boolean; + /** + * Also save the page's cookies. + */ + save_cookies?: boolean; + save_headers?: boolean; + /** + * Also extract and save the page's headings. + */ + save_headings?: boolean; + /** + * Also save the page's rendered HTML. + */ + save_html?: boolean; + save_http_headers?: boolean; + /** + * Also extract and save the list of links found on the page. + */ + save_links?: boolean; + /** + * Also extract and save the page as Markdown alongside the render. + */ + save_markdown?: boolean; + /** + * Also extract and save page metadata (title, description, OpenGraph, etc.). + */ + save_metadata?: boolean; + /** + * Also save the page as MHTML (a single-file web archive). + */ + save_mhtml?: boolean; + /** + * Also save information about the resources the page loaded. + */ + save_resource_info?: boolean; + /** + * Also extract and save structured data on the page (JSON-LD / microdata). + */ + save_structured_data?: boolean; + /** + * Also extract and save the page's tables. + */ + save_tables?: boolean; + scrgb_linear?: boolean; + /** + * When Urlbox decides to split a screenshot into multiple sections, the scroll delay is the time to wait between taking the screenshots of each individual section, in milliseconds. While Urlbox does detect animations, and attempts to wait for them before taking a screenshot, this option could be used to force Urlbox to wait for a certain amount of time after scrolling to the next section, to wait for things like animations to finish. + */ + scroll_delay?: number; + scroll_height?: number; + /** + * Sets how many pixels to scroll when scrolling the page to trigger lazy loading elements. By default, the scroll increment is set to the browser viewport height. Some pages' lazy loading elements only trigger when the scroll increment is smaller than this, however, e.g. 400px. + */ + scroll_increment?: number; + /** + * Scroll, to either an element or to a pixel offset from the top, before taking a screenshot + */ + scroll_to?: string | number; + scroll_to_x?: string | number; + /** + * Ensure that sensitive data remains protected throughout the process. In addition to this option, you need to use one of the storage approaches described on our [`blog post`]( /secure-screenshots ) about secure mode. + */ + secure_mode?: boolean; + /** + * Take a screenshot of the element that matches this selector. By default, if the selector is not found, Urlbox will take a normal viewport screenshot. If you prefer Urlbox to fail the request when the selector is not found, pass `fail_if_selector_missing=true`. + */ + selector?: string; + sharp_stitch?: boolean; + /** + * Shows browser certificate errors for pages with invalid HTTPS certificates. By default, Urlbox ignores certificate errors so pages with expired, self-signed, or otherwise invalid certificates can still render. + * @default "false" + */ + show_certificate_errors?: boolean; + show_seams?: boolean; + show_sections?: boolean; + /** + * Shows a timestamp in a header above the rendered screenshot. Can be paired with show URL. If you're rendering a PDF, you can achieve this with the show_header option. + */ + show_timestamp?: boolean; + /** + * Shows a URL in a header above the rendered screenshot. Can be paired with show timestamp. If you're rendering a PDF, you can achieve this with the show_header option. + */ + show_url?: boolean; + skip_final_delay?: boolean; + /** + * Enabling `skip_scroll` will speed up renders by skipping an initial scroll through the page, which is used to trigger any lazy loading elements. + * @default "false" + */ + skip_scroll?: boolean; + skip_webhooks?: boolean; + solve_captchas?: boolean; + solve_cf_turnstile?: boolean; + tag?: string | string[]; + /** + * The height of the generated thumbnail, in pixels. Omit for a full-size screenshot. + */ + thumb_height?: number; + /** + * The width of the generated thumbnail, in pixels. Omit for a full-size screenshot. + */ + thumb_width?: number; + /** + * Generate up to 5 additional thumbnail images from the same render, each uploaded as its own file alongside the main screenshot. `thumbnails` is independent of the main screenshot's own dimensions, and unrelated to the legacy [`thumb_width`](#thumb_width)/[`thumb_height`](#thumb_height) options above, which resize the primary screenshot itself. + * + * Each entry in the array is an object with the following keys, all optional: + * + * - `key` — a short identifier (max 10 characters) for this thumbnail. Used as its filename suffix, and as its key in the JSON response. + * - `preset` — one of `xs`, `sm`, `md`, `lg`, `xl`, `2xl`, `3xl`, `4xl`, `5xl`, `1/2`, `1/4`, `3/4`. Scales the thumbnail as a percentage of the original screenshot (e.g. `md` is 30%). Takes priority over `size`/`width`/`height`. + * - `size` — a single pixel value (`10`–`2000`) used as both width and height. + * - `width` / `height` — the thumbnail dimensions in pixels (`10`–`2000`). + * - `fit` — how the thumbnail is resized/cropped: `cover`, `contain`, `fill`, `inside` or `outside`. Falls back to [`img_fit`](#img_fit), then `cover`. + * - `bg` — background colour used for letterboxing. Falls back to [`img_bg`](#img_bg), then [`bg_color`](#bg_color), then `black`. + * - `position` — how the image is positioned within its `fit`. Any [`img_position`](#img_position) value, including `attention` and `entropy`. Falls back to `img_position`, then `north`. + * - `suffix` — a custom filename suffix, used when `key` isn't set. + * - `presigned_url` — a presigned URL to upload this specific thumbnail to, instead of Urlbox's own storage. + * + * Each generated thumbnail appears in the JSON response (see [`response_type`](#response_type)) as an entry in a `thumbnails` array, each with a `key`, `location` and `size`. + */ + thumbnails?: ({ + key?: string; + preset?: "xs" | "sm" | "md" | "lg" | "xl" | "2xl" | "3xl" | "4xl" | "5xl" | "1/2" | "1/4" | "3/4"; + size?: number; + width?: number; + height?: number; + fit?: "cover" | "contain" | "fill" | "inside" | "outside"; + bg?: string; + position?: "top" | "right top" | "right" | "right bottom" | "bottom" | "left bottom" | "left" | "left top" | "center" | "centre" | "north" | "northeast" | "east" | "southeast" | "south" | "southwest" | "west" | "northwest" | "entropy" | "attention"; + suffix?: string; + presigned_url?: string; + })[]; + /** + * When `true`, the generated [`thumbnails`](#thumbnails) are returned in the JSON response as an object keyed by each thumbnail's `key`/suffix (with `key` omitted from each value) instead of an array — so you can look a thumbnail up by name rather than by position. + * @default "false" + */ + thumbnails_object?: boolean; + /** + * The amount of time to wait for the requested URL to load, in milliseconds. The timeout value needs to be between 5,000 and 100,000 milliseconds. The default is 30000 or 30 seconds. + * @default "30000" + */ + timeout?: number; + /** + * If a website has no background color set, the image will have a transparent background (PNG/WebP only) + * @default "false" + */ + transparent?: boolean; + /** + * The duration to keep a screenshot or PDF in the cache, in seconds. ttl stands for 'time to live'. The default value is also the maximum value: `2592000` seconds (30 days). + * @default 2592000 + */ + ttl?: number; + turbo?: boolean; + /** + * Emulate the timezone to use when rendering pages. + * + * By default the rendering browser reports a US timezone (`America/New_York` or `America/Los_Angeles`, depending on which of our regions renders your screenshot) so that its clock agrees with where our renderers connect from. Set `tz` explicitly to pin it. Renders on the `stable` engine channel still report UTC until the next promotion of `latest` to `stable` (see [`engine_version`](#engine_version)). + * + * This matters whenever the page derives anything from the browser's local clock: displayed dates and times, opening hours, "expires in" countdowns, and any cookie or token whose lifetime the page calculates client-side. If your renders need to be identical every time, or your page assumes UTC, set `tz=UTC`. + * + * Example: `tz=Europe/London`. A list of timezone ID's can be found here: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + * @default "America/New_York" + */ + tz?: string; + /** + * Pass a unique string such as a UUID, hash or timestamp, to have more control over when to generate a fresh screenshot or PDF. + */ + unique?: string; + urlbox_proxy?: string; + /** + * Save the render directly to the Azure Blob Storage container configured on your project. Azure is not S3-compatible, so it has its own credentials (storage account, container and SAS token) and options - see the [Azure Blob Storage guide](/docs/storage/configure-azure-blob-storage) for setup. + * + * Mutually exclusive with [`use_s3`](#use_s3) - a request setting both fails. + * @default "false" + */ + use_azure?: boolean; + use_chrome?: boolean; + use_chromium?: boolean; + /** + * Use the LLM configuration setup in your project settings, or those passed into the request. + */ + use_llm?: boolean; + /** + * This uses the proxy you have stored on your [`project`]( /dashboard/projects ) in use to make screenshot requests via that server. + */ + use_proxy?: boolean; + /** + * Save the render directly to the S3 (or S3-Compatible) bucket configured on your account. + * + * Mutually exclusive with [`use_azure`](#use_azure) - a request setting both fails. + * @default "false" + */ + use_s3?: boolean; + /** + * Enable stealth mode to reduce automated-browser detection on sites that block headless browsers. + */ + use_stealth?: boolean; + use_tailwind?: boolean; + /** + * Sets the `User-Agent` string for the request. The user agent identifies what browser/device is making the request, which can affect how websites render content. + * + * **Presets:** + * * `random` - Uses a random user-agent to help avoid bot detection + * * `mobile` - Uses a modern iPhone/Safari user-agent string + * * `desktop` - Uses a modern Chrome/macOS user-agent string + * + * **Why use this?** + * - Some websites serve different content based on the user agent (e.g., mobile vs desktop layouts) + * - Certain sites block requests from unknown or bot-like user agents + * - You may want to emulate how a specific browser or crawler sees a page + * + * **Testing your user agent:** + * Try rendering [httpbin.org/user-agent](https://httpbin.org/user-agent) to see exactly what user agent string is being sent. + * + * For a comprehensive list of user agent strings, see [useragents.me](https://www.useragents.me/). + */ + user_agent?: string; + v?: "stable" | "latest" | "lts" | "experimental" | (string & {}); + video_aspect?: number; + video_bitrate?: number; + /** + * Target bitrate of the output video, in bits per second. + */ + video_bits_per_second?: number; + video_chrome_height?: number; + video_codec?: "h264" | "vp9" | "vp8"; + video_crop_w?: number; + video_duration?: number; + /** + * Easing function applied to the scroll motion (for example `ease-in-out`). + */ + video_ease?: "linear.none" | "quadratic.in" | "quadratic.out" | "quadratic.inout" | "cubic.in" | "cubic.out" | "cubic.inout" | "quartic.in" | "quartic.out" | "quartic.inout" | "quintic.in" | "quintic.out" | "quintic.inout" | "sinusoidal.in" | "sinusoidal.out" | "sinusoidal.inout" | "exponential.in" | "exponential.out" | "exponential.inout" | "circular.in" | "circular.out" | "circular.inout" | "elastic.in" | "elastic.out" | "elastic.inout" | "back.in" | "back.out" | "back.inout" | "bounce.in" | "bounce.out" | "bounce.inout"; + /** + * Easing function applied at the end of the scroll. + */ + video_ease_end?: "linear.none" | "quadratic.in" | "quadratic.out" | "quadratic.inout" | "cubic.in" | "cubic.out" | "cubic.inout" | "quartic.in" | "quartic.out" | "quartic.inout" | "quintic.in" | "quintic.out" | "quintic.inout" | "sinusoidal.in" | "sinusoidal.out" | "sinusoidal.inout" | "exponential.in" | "exponential.out" | "exponential.inout" | "circular.in" | "circular.out" | "circular.inout" | "elastic.in" | "elastic.out" | "elastic.inout" | "back.in" | "back.out" | "back.inout" | "bounce.in" | "bounce.out" | "bounce.inout"; + video_ffmpeg?: string | string[]; + /** + * Frames per second of the output video. + */ + video_fps?: number; + /** + * Height of the output video, in pixels. + */ + video_height?: number; + /** + * Adds slight random variation (jitter) to the scroll motion. + */ + video_jitter?: number; + video_method?: "screencast" | "psr" | "extension" | "modern"; + /** + * Pause after scrolling finishes, in milliseconds. + */ + video_postscroll_duration?: number; + /** + * Pause before scrolling starts, in milliseconds. + */ + video_prescroll_duration?: number; + video_preset?: "ultrafast" | "superfast" | "veryfast" | "faster" | "fast" | "medium" | "slow" | "slower" | "veryslow"; + video_quality?: number; + video_refs?: number; + /** + * Rest/pause between scroll segments, in milliseconds. + */ + video_rest_duration?: number; + /** + * Smoothly scroll down the page while recording, section by section, then scroll back to the top. Scroll speed, easing, pauses and distance are controlled by the options below. + * @default "false" + */ + video_scroll?: boolean; + /** + * Whether to scroll back up to the top after reaching the bottom. + */ + video_scroll_back?: boolean; + /** + * Duration of the scroll-back motion, in milliseconds. + */ + video_scroll_back_duration?: number; + /** + * Distance in pixels to scroll the page during the video. + */ + video_scroll_distance?: number; + /** + * How long the scroll takes, in milliseconds. + */ + video_scroll_duration?: number; + video_scroll_offset?: number; + video_scroll_require_sections?: boolean; + /** + * Scroll to and pause on specific page sections in turn while recording, instead of a constant scroll. Each stop is a CSS selector or a `text=...` locator, optionally followed by `;wait=...` and `;duration=...` (over a render link), or given as `{ selector | text, wait, duration, ease, offset }` objects (in a JSON body). See the [Recording videos guide](/docs/videos/recording-videos). + */ + video_scroll_to?: ({ + selector?: string; + text?: string; + wait?: number; + duration?: number; + ease?: "linear.none" | "quadratic.in" | "quadratic.out" | "quadratic.inout" | "cubic.in" | "cubic.out" | "cubic.inout" | "quartic.in" | "quartic.out" | "quartic.inout" | "quintic.in" | "quintic.out" | "quintic.inout" | "sinusoidal.in" | "sinusoidal.out" | "sinusoidal.inout" | "exponential.in" | "exponential.out" | "exponential.inout" | "circular.in" | "circular.out" | "circular.inout" | "elastic.in" | "elastic.out" | "elastic.inout" | "back.in" | "back.out" | "back.inout" | "bounce.in" | "bounce.out" | "bounce.inout"; + offset?: number; + })[]; + video_sections?: number; + video_seek?: number; + /** + * Total duration of the video, in milliseconds (8000 = 8 seconds). Longer videos cost more, so set a sensible cap. + */ + video_time?: number; + video_use_iscroll?: boolean; + video_warmup?: boolean; + /** + * Width of the output video, in pixels. + */ + video_width?: number; + viewport_clip?: boolean; + /** + * Waits for the element specified by this selector to be present in the DOM before taking a screenshot or PDF. + * + * By default, Urlbox will take a screenshot or PDF if the `wait_for` element is not found after waiting for the time specified by the [`wait_timeout`](#wait_timeout) option. + * + * If you prefer Urlbox to fail the request when the `wait_for` element is not found, pass [`fail_if_selector_missing=true`](#fail_if_selector_missing) + */ + wait_for?: string; + /** + * Whether the element specified by wait_for should be visible or just present in the DOM. Visible means the element is attached to the DOM and has opacity > 0 and its width and height > 0 and doesn't have `visibility:hidden` set. + * @default "attached" + */ + wait_for_state?: "attached" | "visible"; + wait_for_x?: string; + /** + * The amount of time to wait for the [`wait_for`](#wait_for) element to appear, or the [`wait_to_leave`](#wait_to_leave) element to leave before continuing, in milliseconds. + * @default 30000 + */ + wait_timeout?: number; + /** + * Waits for the element specified by this selector to be absent from the DOM before taking a screenshot or PDF. + * + * A typical use-case would be waiting for loading spinners to be absent before taking a screenshot. + * + * By default, Urlbox will take a screenshot or PDF if the `wait_to_leave` element is still present after the time specified by the [`wait_timeout`](#wait_timeout) option. + * + * If you prefer Urlbox to fail the request when the `wait_to_leave` element is still present, pass [`fail_if_selector_present=true`](#fail_if_selector_present) + */ + wait_to_leave?: string; + wait_to_leave_pierce?: boolean; + wait_to_leave_state?: "detached" | "hidden"; + wait_to_leave_x?: string; + /** + * Waits until the specified DOM event has fired before capturing a render. + * + * The available options are: + * + * * `domloaded` (the `DOMContentLoaded` event is fired) + * * `mostrequestsfinished` (consider navigation to be finished when there are no more than 2 network connections for at least 500 ms) + * * `requestsfinished` (there are no more than 0 network connections for at least 500 ms) + * * `loaded` (the `load` event is fired) + * @default "loaded" + */ + wait_until?: "domloaded" | "mostrequestsfinished" | "requestsfinished" | "loaded" | "smart"; + watermark?: boolean; + web_bot_auth?: boolean; + web_bot_auth_optional?: boolean; + /** + * A URL that Urlbox will POST the render result to once the render completes, when using the asynchronous render endpoint. Verify the request signature with your project's webhook secret. + */ + webhook_url?: string; + /** + * The viewport width of the browser, in pixels. + * @default 1280 + */ + width?: number; + width_from?: string; + wrap?: string; +} + +export interface UrlInput { + /** + * The URL or domain of the website you want to screenshot. We will automatically prepend `http://` if it is missing. + */ + url: string; + html?: never; +} + +export interface HtmlInput { + /** + * The HTML you want to render. + */ + html: string; + url?: never; +} diff --git a/src/responses.ts b/src/responses.ts new file mode 100644 index 0000000..1260562 --- /dev/null +++ b/src/responses.ts @@ -0,0 +1,161 @@ +export interface ThumbnailResult { + key: string; + location: string; + size: number; +} + +export interface SliceResult { + index: number; + offset_y: number; + width: number; + height: number; + url: string; + size?: number; +} + +/** The target page's HTTP response. `headers` only with `save_headers`. */ +export interface RenderResponseInfo { + statusCode: number; + statusCodeInitial?: number; + /** `Key=Value` strings. */ + headers?: string[]; + urlRequested?: string; + urlResolved?: string; + urlResolvedInitial?: string; +} + +export interface BlockDetectionResult { + blocked: boolean; + score: number; + provider: string; + signals: string[]; +} + +export interface UnblockSummary { + blocked: boolean; + unblocked: boolean; + attempts: number; + rung: number | null; + rungOptions: Record<string, unknown> | null; +} + +/** The JSON body of a successful render (`POST /v1/render/sync`). */ +export interface RenderResult { + /** Where the render was uploaded. */ + renderUrl: string; + /** Size of the render in bytes. */ + size: number; + renderTime?: number; + queueTime?: number; + bandwidth?: number; + totalBandwidth?: number; + width?: number; + height?: number; + pageWidth?: number; + pageHeight?: number; + requestDomains?: string[]; + /** An array, or an object keyed by thumbnail key with `thumbnails_object`. */ + thumbnails?: ThumbnailResult[] | Record<string, Omit<ThumbnailResult, "key">>; + slices?: SliceResult[]; + htmlUrl?: string; + mhtmlUrl?: string; + metadataUrl?: string; + headingsUrl?: string; + tablesUrl?: string; + structuredDataUrl?: string; + markdownUrl?: string; + custom_id?: string; + metadata?: Record<string, unknown>; + llm?: unknown; + timestamp?: string; + certifiedHash?: string; + hashedOptions?: string; + response?: RenderResponseInfo; + /** Credits charged, for accounts that have credit reporting enabled. */ + creditsUsed?: number; + /** Renders drawn from your allowance by this request. */ + renders?: number; + /** Present while the account's payment is failing. */ + warning?: string; + blockDetection?: BlockDetectionResult; + unblock?: UnblockSummary; +} + +/** `POST /v1/render/async` response (HTTP 201). */ +export interface AsyncRenderCreated { + status: "created"; + renderId: string; + /** Poll this URL (or call `getRender(renderId)`) for the result. */ + statusUrl: string; + warning?: string; +} + +export interface RenderPending { + renderId: string; + status: "created" | "retrying"; + statusUrl: string; + /** Set when a stalled render is being retried. */ + stalled?: boolean; +} + +export interface RenderSucceeded extends RenderResult { + renderId: string; + status: "succeeded"; +} + +export interface RenderFailed { + renderId: string; + status: "failed"; + reason?: string; + renderTime?: number; + queueTime?: number; + debugScreenshot?: { location: string; size?: number }; + /** The target page's final HTTP status, when the page loaded. */ + statusCode?: number; + statusCodeInitial?: number; + headers?: string[]; + urlRequested?: string; + urlResolved?: string; + urlResolvedInitial?: string; +} + +export interface RenderNotFound { + renderId: string; + status: "not-found"; +} + +/** `GET /v1/render/:renderId`, discriminated on `status`. */ +export type RenderStatus = + RenderPending | RenderSucceeded | RenderFailed | RenderNotFound; + +export interface WebhookMeta { + /** ISO 8601. */ + startTime: string; + /** ISO 8601. */ + endTime: string; +} + +export interface WebhookRenderSucceeded { + event: "render.succeeded"; + renderId: string; + customId?: string; + result: RenderResult & { renders: number }; + meta: WebhookMeta; +} + +export interface WebhookRenderFailed { + event: "render.failed"; + renderId: string; + customId?: string; + error: { message: string; code?: string }; + meta: WebhookMeta; + result?: { + renders: number; + renderTime?: number; + queueTime?: number; + response?: RenderResponseInfo; + debugScreenshot?: { location: string; size?: number }; + }; +} + +export type WebhookEvent = WebhookRenderSucceeded | WebhookRenderFailed; diff --git a/src/types.ts b/src/types.ts index 17449ce..103061c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,386 +1,80 @@ -export type RenderOptions = UrlOptions | HtmlOptions; - -type UrlOptions = CommonOptions & { - url: string; - html?: never; -}; - -type HtmlOptions = CommonOptions & { - html: string; - url?: never; -}; - -type CommonOptions = Partial<{ - full_page: boolean; - - format: (typeof formats)[number]; - pdf_page_size: (typeof pdfPageSize)[number]; - pdf_orientation: (typeof pdfOrientationEnum)[number]; - wait_until: (typeof waitUntil)[number]; - media: (typeof mediaOptionsEnum)[number]; - pdf_margin: (typeof pdfMarginEnum)[number]; - s3_storageclass: (typeof storageClasses)[number]; - color_profile: (typeof colorProfiles)[number]; - - full_width: boolean; - disable_animations: boolean; - force: boolean; - retina: boolean; - disable_js: boolean; - freeze_fixed: boolean; - flash: boolean; - pdf_background: boolean; - transparent: boolean; - hide_cookie_banners: boolean; - block_ads: boolean; - block_images: boolean; - block_scripts: boolean; - block_frames: boolean; - block_fonts: boolean; - block_medias: boolean; - block_styles: boolean; - block_fetch: boolean; - block_xhr: boolean; - block_sockets: boolean; - block_manifests: boolean; - block_other: boolean; - block_pings: boolean; - click_accept: boolean; - accept_cookies: boolean; - skip_scroll: boolean; - fail_on_4xx: boolean; - fail_on_5xx: boolean; - fail_if_selector_missing: boolean; - fail_if_selector_present: boolean; - fail_if_captcha: boolean; - save_html: boolean; - metadata: boolean; - save_metadata: boolean; - save_markdown: boolean; - kill_popups: boolean; - lazyload: boolean; - sharp_stitch: boolean; - allow_infinite: boolean; - detect_full_height: boolean; - allow_coin: boolean; - solve_captchas: boolean; - show_sections: boolean; - show_seams: boolean; - watermark: boolean; - pdf_fit_to_page: boolean; - use_s3: boolean; - s3_force_path_style: boolean; - use_tailwind: boolean; - use_chrome: boolean; - use_chromium: boolean; - no_upload: boolean; - from_html: boolean; - fix_full_height: boolean; - skip_final_delay: boolean; - turbo: boolean; - full_html: boolean; - save_mhtml: boolean; - skip_webhooks: boolean; - dark_mode: boolean; - reduced_motion: boolean; - gpu: boolean; - json: boolean; - lightweight: boolean; - press_escape: boolean; - fragment: boolean; - display_p3: boolean; - rec2020: boolean; - scrgb_linear: boolean; - hdr10: boolean; - readable: boolean; - retry_on_nav_error: boolean; - continue_on_nav_error: boolean; - s3_private_bucket: boolean; - disable_web_security: boolean; - - unique: string; - user_agent: string; - wrap: string; - bg_color: string; - hide_selector: string; - hide_selector_x: string; - highlight: string; - highlightfg: string; - highlightbg: string; - selector: string; - css: string; - js: string; - s3_key: string; - s3_secret: string; - s3_bucket: string; - s3_path: string; - s3_region: string; - s3_endpoint: string; - s3_presigned_url: string; - - cdn_host: string; - wait_for: string; - wait_for_x: string; - wait_to_leave: string; - wait_to_leave_x: string; - webhook_url: string; - base_url: string; - width_from: string; - height_from: string; - blend_mode: string; - proxy: string; - clip: string; - tz: string; - custom_data_variable: string; - pdf_header: string; - pdf_footer: string; - pdf_page_range: string; - img_pad: string; - full_page_mode: (typeof fullPageMode)[number]; - - quality: number; - delay: number; - timeout: number; - scroll_delay: number; - scroll_to: string | number; - scroll_to_x: string | number; - width: number; - height: number; - thumb_width: number; - thumb_height: number; - - crop_width: number; - max_height: number; - max_width: number; - max_section_height: number; - max_section_width: number; - max_sections: number; - max_xsections: number; - max_scroll_page_time: number; - ttl: number; - wait_timeout: number; - redirect_after: number; - scroll_increment: number; - pdf_page_width: number; - pdf_page_height: number; - pdf_scale: number; - pdf_dpi: number; - pdf_margin_top: number; - pdf_margin_bottom: number; - pdf_margin_left: number; - pdf_margin_right: number; - longitude: number; - latitude: number; - accuracy: number; - - header: string | string[]; - cookie: string | string[]; - click: string | string[]; - click_all: string | string[]; - click_x: string | string[]; - click_all_x: string | string[]; - hover: string | string[]; - hover_x: string | string[]; - block_urls: string | string[]; - - video_width: number; - video_height: number; - video_quality: number; - video_preset: (typeof videoPresets)[number]; - video_bitrate: number; - video_bits_per_second: number; - video_time: number; - video_aspect: number; - video_fps: number; - video_ease: (typeof easingFunctions)[number]; - video_ease_end: (typeof easingFunctions)[number]; - video_prescroll_duration: number; - video_scroll_duration: number; - video_rest_duration: number; - video_scroll_back_duration: number; - video_postscroll_duration: number; - video_scroll_distance: number; - video_scroll: boolean; - video_chrome_height: number; - video_crop_w: number; - video_jitter: number; - - disable_mql: boolean; - disable_resize: boolean; - disable_events: string | string[]; - optimise: boolean; - skip_fonts: boolean; - scroll_height: number; - accept_lang: string; - authorization: string; - pdf_show_header: boolean; - pdf_show_footer: boolean; - - pdf_auto_crop: boolean; - download: string; - no_suffix: boolean; - referer: string; - referrer: string; - disable_ligatures: boolean; - accept_dialogs: boolean; - custom_id: any; - renderer: string; - skip_wait: boolean | undefined; - skip_wait_for_modifications: any; - skip_wait_for_images: any; - watermark_url: string; - watermark_width: string; - watermark_height: string; - watermark_bgcolor: string; - watermark_opacity: number; - watermark_blend: string; - watermark_repeat: string; - watermark_position: string; - watermark_angle: string; - watermark_style: any; - img_fit: "cover" | "contain" | "fill" | "inside" | "outside"; - img_bg: string; - img_position: - | "top" - | "right top" - | "right" - | "right bottom" - | "bottom" - | "left bottom" - | "left" - | "left top" - | "center" - | "centre" - | "north" - | "northeast" - | "east" - | "southeast" - | "south" - | "southwest" - | "west" - | "northwest" - | "center" - | "centre" - | "entropy" - | "attention"; - srgb: boolean; - rgb: boolean; - gamma24: boolean; - response_type: "json" | "jsondebug" | "binary" | "base64" | "none"; - global_timeout: number; -}>; - -const formats = [ - "png", - "jpg", - "jpeg", - "pdf", - "avif", - "mp4", - "webm", - "webp", - "html", - "mhtml", - "svg", - "md", -] as const; - -const pdfPageSize = [ - "a4", - "a0", - "a1", - "a2", - "a3", - "a4", - "a5", - "a6", - "legal", - "ledger", - "letter", - "tabloid", -] as const; - -const pdfOrientation = ["portrait", "landscape"] as const; - -const storageClasses = [ - "standard", - "reduced_redundancy", - "standard_ia", - "onezone_ia", - "intelligent_tiering", - "glacier", - "deep_archive", - "outposts", -] as const; - -const colorProfiles = [ - "default", - "srgb", - "dp3", - "rec2020", - "scrgblinear", - "hdr10", - "colorspingamma24", -] as const; - -const waitUntil = [ - "domloaded", - "mostrequestsfinished", - "requestsfinished", - "loaded", -] as const; - -const pdfOrientationEnum = ["portrait", "landscape"] as const; - -const pdfMarginEnum = ["default", "none", "minimum"] as const; - -const mediaOptionsEnum = ["print", "screen"] as const; -const fullPageMode = ["stitch", "native"] as const; -const videoPresets = [ - "ultrafast", - "superfast", - "veryfast", - "faster", - "fast", - "medium", - "slow", - "slower", - "veryslow", -] as const; - -const easingFunctions = [ - "linear.none", - "linear.in", - "linear.out", - "linear.inout", - "quadratic.in", - "quadratic.out", - "quadratic.inout", - "cubic.in", - "cubic.out", - "cubic.inout", - "quartic.in", - "quartic.out", - "quartic.inout", - "quintic.in", - "quintic.out", - "quintic.inout", - "sinusoidal.in", - "sinusoidal.out", - "sinusoidal.inout", - "exponential.in", - "exponential.out", - "exponential.inout", - "circular.in", - "circular.out", - "circular.inout", - "elastic.in", - "elastic.out", - "elastic.inout", - "back.in", - "back.out", - "back.inout", - "bounce.in", - "bounce.out", - "bounce.inout", -] as const; +import type { + GeneratedRenderOptionFields, + HtmlInput, + UrlInput, +} from "./render-options.generated.js"; + +type LegacyEasing = "linear.in" | "linear.out" | "linear.inout"; + +/** + * Options accepted by earlier releases of this package that the API's option + * schema no longer lists. They are kept so existing code keeps compiling. + */ +interface LegacyRenderOptionFields { + /** Passed through to the API as `custom_id`; echoed back on the result. */ + custom_id?: any; + /** @deprecated No longer part of the render options schema. */ + disable_mql?: boolean; + /** @deprecated No longer part of the render options schema. */ + disable_resize?: boolean; + /** @deprecated No longer part of the render options schema. */ + disable_events?: string | string[]; + /** @deprecated No longer part of the render options schema. */ + optimise?: boolean; + /** @deprecated No longer part of the render options schema. */ + skip_fonts?: boolean; + /** @deprecated No longer part of the render options schema. */ + referer?: string; + /** @deprecated No longer part of the render options schema. */ + referrer?: string; + /** @deprecated No longer part of the render options schema. */ + accept_dialogs?: boolean; + /** @deprecated No longer part of the render options schema. */ + renderer?: string; + /** @deprecated No longer part of the render options schema. */ + skip_wait?: boolean | undefined; + /** @deprecated No longer part of the render options schema. */ + skip_wait_for_modifications?: any; + /** @deprecated No longer part of the render options schema. */ + skip_wait_for_images?: any; + /** @deprecated No longer part of the render options schema. */ + watermark_url?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_width?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_height?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_bgcolor?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_opacity?: number; + /** @deprecated No longer part of the render options schema. */ + watermark_blend?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_repeat?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_position?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_angle?: string; + /** @deprecated No longer part of the render options schema. */ + watermark_style?: any; + /** @deprecated No longer part of the render options schema. */ + srgb?: boolean; + /** @deprecated No longer part of the render options schema. */ + rgb?: boolean; + /** @deprecated No longer part of the render options schema. */ + gamma24?: boolean; + /** @deprecated No longer part of the render options schema. */ + global_timeout?: number; + video_ease?: GeneratedRenderOptionFields["video_ease"] | LegacyEasing; + video_ease_end?: GeneratedRenderOptionFields["video_ease_end"] | LegacyEasing; +} + +type CommonOptions = Omit< + GeneratedRenderOptionFields, + keyof LegacyRenderOptionFields +> & + LegacyRenderOptionFields; + +/** Render options: either `url` or `html` is required, plus any other option. */ +export type RenderOptions = + (CommonOptions & UrlInput) | (CommonOptions & HtmlInput); diff --git a/src/webhooks.test.ts b/src/webhooks.test.ts new file mode 100644 index 0000000..ce85206 --- /dev/null +++ b/src/webhooks.test.ts @@ -0,0 +1,203 @@ +import { createHmac } from "node:crypto"; +import { describe, expect, expectTypeOf, test } from "vitest"; +import { + parseWebhook, + UrlboxWebhookSignatureError, + verifyWebhookSignature, +} from "./index.js"; +import type { WebhookEvent } from "./index.js"; + +const secret = "whsec_test"; +const now = new Date("2026-10-21T12:00:00Z"); +const t = Math.floor(now.getTime() / 1000); + +const payload = { + event: "render.succeeded", + renderId: "a1b2c3", + result: { + renderUrl: "https://renders.urlbox.com/x.png", + size: 1234, + renders: 1, + }, + meta: { + startTime: "2026-10-21T11:59:58.000Z", + endTime: "2026-10-21T12:00:00.000Z", + }, +}; +const rawBody = JSON.stringify(payload); + +const hmac = (key: string, timestamp: number, body: string) => + createHmac("sha256", key).update(`${timestamp}.${body}`).digest("hex"); + +const header = (keys: string[], timestamp = t, body = rawBody) => + `t=${timestamp},${keys.map((k) => `sha256=${hmac(k, timestamp, body)}`).join(",")}`; + +describe("verifyWebhookSignature", () => { + test("accepts a valid signature", () => { + expect( + verifyWebhookSignature({ + header: header([secret]), + rawBody, + secret, + now, + }), + ).toBe(true); + }); + + test("accepts a Buffer body", () => { + expect( + verifyWebhookSignature({ + header: header([secret]), + rawBody: Buffer.from(rawBody), + secret, + now, + }), + ).toBe(true); + }); + + test("accepts any signature in a rotation header", () => { + const rotating = header(["whsec_old", secret]); + expect( + verifyWebhookSignature({ header: rotating, rawBody, secret, now }), + ).toBe(true); + expect( + verifyWebhookSignature({ + header: rotating, + rawBody, + secret: "whsec_old", + now, + }), + ).toBe(true); + }); + + test("accepts when any of several secrets matches", () => { + expect( + verifyWebhookSignature({ + header: header([secret]), + rawBody, + secret: ["whsec_new", secret], + now, + }), + ).toBe(true); + }); + + test("rejects the wrong secret", () => { + expect( + verifyWebhookSignature({ + header: header([secret]), + rawBody, + secret: "nope", + now, + }), + ).toBe(false); + }); + + test("rejects a tampered body", () => { + expect( + verifyWebhookSignature({ + header: header([secret]), + rawBody: rawBody.replace("1234", "1235"), + secret, + now, + }), + ).toBe(false); + }); + + test("rejects an expired timestamp", () => { + const old = t - 301; + expect( + verifyWebhookSignature({ + header: header([secret], old), + rawBody, + secret, + now, + }), + ).toBe(false); + expect( + verifyWebhookSignature({ + header: header([secret], old), + rawBody, + secret, + now, + toleranceSeconds: 600, + }), + ).toBe(true); + }); + + test("tolerance 0 disables the timestamp check", () => { + const ancient = t - 86_400 * 365; + expect( + verifyWebhookSignature({ + header: header([secret], ancient), + rawBody, + secret, + now, + toleranceSeconds: 0, + }), + ).toBe(true); + }); + + test("rejects missing or malformed headers", () => { + for (const bad of [ + undefined, + null, + "", + `sha256=${hmac(secret, t, rawBody)}`, + `t=${t}`, + `t=${t},sha256=abc`, + `t=abc,sha256=${hmac(secret, t, rawBody)}`, + ]) { + expect( + verifyWebhookSignature({ header: bad, rawBody, secret, now }), + ).toBe(false); + } + }); +}); + +describe("parseWebhook", () => { + test("returns the typed payload", () => { + const event = parseWebhook({ + header: header([secret]), + rawBody, + secret, + now, + }); + expectTypeOf(event).toEqualTypeOf<WebhookEvent>(); + expect(event).toEqual(payload); + if (event.event === "render.succeeded") { + expectTypeOf(event.result.renderUrl).toEqualTypeOf<string>(); + expect(event.result.renders).toBe(1); + } else { + expectTypeOf(event.error.message).toEqualTypeOf<string>(); + throw new Error("expected render.succeeded"); + } + }); + + test("parses a render.failed payload", () => { + const failed = JSON.stringify({ + event: "render.failed", + renderId: "a1b2c3", + customId: "order-42", + error: { message: "Timed out", code: "Timeout" }, + meta: payload.meta, + result: { renders: 0 }, + }); + const event = parseWebhook({ + header: header([secret], t, failed), + rawBody: failed, + secret, + now, + }); + expect(event.event).toBe("render.failed"); + if (event.event === "render.failed") { + expect(event.error.code).toBe("Timeout"); + expect(event.customId).toBe("order-42"); + } + }); + + test("throws UrlboxWebhookSignatureError when invalid", () => { + expect(() => + parseWebhook({ header: header(["other"]), rawBody, secret, now }), + ).toThrow(UrlboxWebhookSignatureError); + }); +}); diff --git a/src/webhooks.ts b/src/webhooks.ts new file mode 100644 index 0000000..b88c90a --- /dev/null +++ b/src/webhooks.ts @@ -0,0 +1,90 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { UrlboxWebhookSignatureError } from "./errors.js"; +import type { WebhookEvent } from "./responses.js"; + +const DEFAULT_TOLERANCE_SECONDS = 300; +const HEX_SHA256 = /^[0-9a-f]{64}$/i; + +export interface VerifyWebhookArgs { + /** The `X-Urlbox-Signature` header value. */ + header: string | null | undefined; + /** The request body exactly as received, before any JSON parsing. */ + rawBody: string | Uint8Array; + /** Your webhook secret, or several while rotating. */ + secret: string | string[]; + /** Maximum age of the signature timestamp. Default 300; 0 disables the check. */ + toleranceSeconds?: number; + /** Override the current time (for tests). */ + now?: Date; +} + +const parseHeader = (header: string) => { + let timestamp: number | undefined; + const signatures: Buffer[] = []; + for (const part of header.split(",")) { + const index = part.indexOf("="); + if (index === -1) continue; + const key = part.slice(0, index).trim(); + const value = part.slice(index + 1).trim(); + if (key === "t" && /^\d+$/.test(value)) { + timestamp = Number(value); + } else if (key === "sha256" && HEX_SHA256.test(value)) { + signatures.push(Buffer.from(value, "hex")); + } + } + return { timestamp, signatures }; +}; + +const sign = ( + secret: string, + timestamp: number, + rawBody: string | Uint8Array, +) => + createHmac("sha256", secret).update(`${timestamp}.`).update(rawBody).digest(); + +/** + * Checks an `X-Urlbox-Signature` header (`t=<unix>,sha256=<hex>[,sha256=<hex>…]`) + * against the raw request body. Returns true when any signature matches any of + * the given secrets and the timestamp is within tolerance. + */ +export function verifyWebhookSignature({ + header, + rawBody, + secret, + toleranceSeconds = DEFAULT_TOLERANCE_SECONDS, + now = new Date(), +}: VerifyWebhookArgs): boolean { + if (!header) return false; + const { timestamp, signatures } = parseHeader(header); + if (timestamp === undefined || signatures.length === 0) return false; + + if (toleranceSeconds > 0) { + const age = Math.abs(Math.floor(now.getTime() / 1000) - timestamp); + if (age > toleranceSeconds) return false; + } + + const secrets = (Array.isArray(secret) ? secret : [secret]).filter(Boolean); + return secrets.some((s) => { + const expected = sign(s, timestamp, rawBody); + return signatures.some( + (candidate) => + candidate.length === expected.length && + timingSafeEqual(candidate, expected), + ); + }); +} + +/** + * Verifies the signature and returns the parsed webhook payload. + * Throws `UrlboxWebhookSignatureError` when the signature does not verify. + */ +export function parseWebhook(args: VerifyWebhookArgs): WebhookEvent { + if (!verifyWebhookSignature(args)) { + throw new UrlboxWebhookSignatureError(); + } + const text = + typeof args.rawBody === "string" + ? args.rawBody + : new TextDecoder().decode(args.rawBody); + return JSON.parse(text) as WebhookEvent; +} diff --git a/tsconfig-base.json b/tsconfig-base.json index 316bc5a..85f021c 100644 --- a/tsconfig-base.json +++ b/tsconfig-base.json @@ -6,7 +6,9 @@ "declaration": true, "esModuleInterop": true, "inlineSourceMap": false, - "lib": ["esnext"], + "lib": [ + "esnext" + ], "listEmittedFiles": false, "listFiles": false, "moduleResolution": "node", @@ -17,9 +19,17 @@ "skipLibCheck": true, "strict": true, "traceResolution": false, - "types": ["node"] + "types": [ + "node" + ] }, "compileOnSave": false, - "exclude": ["node_modules", "dist"], - "include": ["src"] + "exclude": [ + "node_modules", + "dist", + "src/**/*.test.ts" + ], + "include": [ + "src" + ] } diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..ae10f2e --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1 @@ +{"extends":"./tsconfig-base.json","compilerOptions":{"noEmit":true,"module":"esnext"},"exclude":["node_modules","dist"]} \ No newline at end of file