From 2fb6c57046c05b880ae93ae35976c8d587b5c05f Mon Sep 17 00:00:00 2001 From: erikengervall <5392475+erikengervall@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:58:47 +0000 Subject: [PATCH 1/2] feat(hosted): let callers choose the Parse mode --- README.md | 2 +- node/README.md | 8 ++++- node/anydoc.d.ts | 6 ++++ node/anydoc.js | 15 +++++++-- node/cli.js | 19 ++++++++++-- node/test.mjs | 62 ++++++++++++++++++++++++++++++++++--- python/README.md | 6 ++++ python/anydoc/__init__.py | 39 +++++++++++++++++------ python/tests/test_anydoc.py | 43 ++++++++++++++++++++++--- 9 files changed, 175 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 760d3f18..2a920ec8 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ anydoc reads text-based PDFs locally but does no OCR, so a PDF with scanned or i | Node | `toMarkdown('scan.pdf', { ocr: 'hosted' })` | `apiKey` | | Python | `anydoc.to_markdown("scan.pdf", ocr="hosted")` | `api_key` | -Only documents that need OCR leave the machine, and the whole document goes, since Parse has no page selection. If Parse cannot convert it, Node rejects with `code: 'hosted'` and Python raises `HostedError`. `--api-url`, `apiUrl` and `api_url`, else `FIRECRAWL_API_URL`, point at another Parse deployment. The Rust crate has no `ocr` option and never makes network calls. +Only documents that need OCR leave the machine, and the whole document goes, since Parse has no page selection. If Parse cannot convert it, Node rejects with `code: 'hosted'` and Python raises `HostedError`. `--api-url`, `apiUrl` and `api_url`, else `FIRECRAWL_API_URL`, point at another Parse deployment. `--parse-mode`, `parseMode` and `parse_mode` pick the Parse PDF mode: `auto` (the default) extracts text and falls back to OCR, `fast` extracts text only, `ocr` OCRs every page. The Rust crate has no `ocr` option and never makes network calls. ## Features diff --git a/node/README.md b/node/README.md index 74500241..12ee0ce4 100644 --- a/node/README.md +++ b/node/README.md @@ -63,7 +63,13 @@ anydoc converts locally and does not do OCR, so a PDF with scanned or image-only const markdown = await toMarkdown('scan.pdf', { ocr: 'hosted' }); ``` -On the CLI, `anydoc scan.pdf --ocr hosted`. +`parseMode` picks the Parse PDF mode: `'auto'` (the default) extracts text and falls back to OCR, `'fast'` extracts text only, `'ocr'` OCRs every page. Any other value rejects with `code: 'hosted'` before anything is sent. + +```js +const markdown = await toMarkdown('scan.pdf', { ocr: 'hosted', parseMode: 'ocr' }); +``` + +On the CLI, `anydoc scan.pdf --ocr hosted`, and `--parse-mode ocr` to pick the mode. ## Errors diff --git a/node/anydoc.d.ts b/node/anydoc.d.ts index b3c2f216..88577535 100644 --- a/node/anydoc.d.ts +++ b/node/anydoc.d.ts @@ -10,6 +10,12 @@ export interface ConvertOptions { * machine. */ ocr?: 'reject' | 'hosted' + /** + * Firecrawl Parse PDF mode for `hosted`: `auto` (the default) extracts text + * and falls back to OCR, `fast` extracts text only, `ocr` OCRs every page. + * Any other value rejects with `code: 'hosted'` before anything is sent. + */ + parseMode?: 'auto' | 'fast' | 'ocr' /** Firecrawl API key for `hosted`, else `FIRECRAWL_API_KEY`, else keyless. */ apiKey?: string /** Firecrawl API URL for `hosted`, else `FIRECRAWL_API_URL`, else `https://api.firecrawl.dev`. */ diff --git a/node/anydoc.js b/node/anydoc.js index c4b38844..af620c75 100644 --- a/node/anydoc.js +++ b/node/anydoc.js @@ -8,13 +8,16 @@ const { version } = require('./package.json') const API_URL = 'https://api.firecrawl.dev' const TIMEOUT_MS = 300_000 +const PARSE_MODES = ['auto', 'fast', 'ocr'] /** * Convert a document file to Markdown. `options.ocr` decides what happens to * a PDF whose pages need OCR: `'reject'` (the default) rejects with - * `needsOcr`, `'hosted'` sends the document to Firecrawl Parse instead. + * `needsOcr`, `'hosted'` sends the document to Firecrawl Parse instead, in + * `options.parseMode` (`'auto'` by default). */ async function toMarkdown(path, options) { + checkParseMode(options) try { return await native.toMarkdown(path) } catch (error) { @@ -25,6 +28,7 @@ async function toMarkdown(path, options) { /** `toMarkdown` for bytes; `options` as there. */ async function toMarkdownBytes(bytes, format, options) { + checkParseMode(options) try { return await native.toMarkdownBytes(bytes, format) } catch (error) { @@ -33,6 +37,13 @@ async function toMarkdownBytes(bytes, format, options) { } } +function checkParseMode(options) { + const mode = options?.parseMode + if (mode !== undefined && !PARSE_MODES.includes(mode)) { + throw hostedError(`invalid parseMode '${mode}'; expected one of: ${PARSE_MODES.join(', ')}`) + } +} + function sendsToHosted(error, options) { return error.code === 'needsOcr' && options?.ocr === 'hosted' } @@ -43,7 +54,7 @@ async function parseHosted(bytes, filename, options) { const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY const apiUrl = options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? API_URL const url = `${apiUrl.replace(/\/$/, '')}/v2/parse` - const parse = { parsers: [{ type: 'pdf', mode: 'auto' }], origin: `anydoc@${version}` } + const parse = { parsers: [{ type: 'pdf', mode: options.parseMode ?? 'auto' }], origin: `anydoc@${version}` } const body = new FormData() body.append('options', JSON.stringify(parse)) body.append('file', new Blob([bytes], { type: 'application/pdf' }), filename) diff --git a/node/cli.js b/node/cli.js index 5d7cd34c..44fcd9dd 100644 --- a/node/cli.js +++ b/node/cli.js @@ -24,6 +24,9 @@ Options: --ocr What to do with a PDF whose pages need OCR: reject (default) exits 3; hosted sends the document to Firecrawl Parse + --parse-mode Firecrawl Parse PDF mode for --ocr hosted: auto + (default) falls back to OCR, fast extracts text + only, ocr OCRs every page --api-key Firecrawl API key for --ocr hosted, else FIRECRAWL_API_KEY, else keyless --api-url Firecrawl API URL for --ocr hosted, else @@ -52,6 +55,7 @@ Examples: ` const OCR_MODES = ['reject', 'hosted'] +const PARSE_MODES = ['auto', 'fast', 'ocr'] const USAGE_ERROR = 2 const CONVERSION_ERROR = 1 @@ -63,7 +67,7 @@ function fail(code, message) { } function parseArgs(argv) { - const args = { input: null, output: null, format: null, ocr: null, apiKey: null, apiUrl: null } + const args = { input: null, output: null, format: null, ocr: null, parseMode: null, apiKey: null, apiUrl: null } let positionalOnly = false for (let i = 0; i < argv.length; i++) { let arg = argv[i] @@ -114,6 +118,12 @@ function parseArgs(argv) { fail(USAGE_ERROR, `invalid --ocr '${args.ocr}'; expected one of: ${OCR_MODES.join(', ')}`) } break + case '--parse-mode': + args.parseMode = value() + if (!PARSE_MODES.includes(args.parseMode)) { + fail(USAGE_ERROR, `invalid --parse-mode '${args.parseMode}'; expected one of: ${PARSE_MODES.join(', ')}`) + } + break case '--api-key': args.apiKey = value() break @@ -156,7 +166,12 @@ async function main() { } } - const options = { ocr: args.ocr ?? undefined, apiKey: args.apiKey ?? undefined, apiUrl: args.apiUrl ?? undefined } + const options = { + ocr: args.ocr ?? undefined, + parseMode: args.parseMode ?? undefined, + apiKey: args.apiKey ?? undefined, + apiUrl: args.apiUrl ?? undefined, + } let markdown try { if (args.input === '-') { diff --git a/node/test.mjs b/node/test.mjs index 0d4f6a4b..c7028c9b 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -101,15 +101,18 @@ test('the package exports everything the native binding does', async () => { }) // A stand-in for api.firecrawl.dev that answers every request with `reply` -// and records each hit as [path, whether a PDF came with it]. Runs `run` -// keyless, whatever the environment. +// and records each hit as [path, whether a PDF came with it], and the Parse +// options each one sent. Runs `run` keyless, whatever the environment. async function withHostedStub(reply, run) { const hits = [] + const sent = [] const server = createServer((request, response) => { const chunks = [] request.on('data', (chunk) => chunks.push(chunk)) request.on('end', () => { - hits.push([request.url, Buffer.concat(chunks).includes('%PDF-')]) + const payload = Buffer.concat(chunks) + hits.push([request.url, payload.includes('%PDF-')]) + sent.push(parseOptions(payload.toString('latin1'))) response.writeHead(reply.status, { 'content-type': 'application/json' }) response.end(JSON.stringify(reply.body)) }) @@ -120,7 +123,7 @@ async function withHostedStub(reply, run) { process.env.FIRECRAWL_API_URL = url delete process.env.FIRECRAWL_API_KEY try { - await run(hits, url) + await run(hits, url, sent) } finally { for (const [name, value] of Object.entries(saved)) { if (value === undefined) delete process.env[name] @@ -130,6 +133,11 @@ async function withHostedStub(reply, run) { } } +function parseOptions(payload) { + const start = payload.indexOf('name="options"\r\n\r\n') + 'name="options"\r\n\r\n'.length + return JSON.parse(payload.slice(start, payload.indexOf('\r\n--', start))) +} + const HOSTED = { status: 200, body: { success: true, data: { markdown: '# Read by the hosted parser\n' } } } test("ocr: 'hosted' sends a pdf with scanned pages to Firecrawl Parse, and nothing else", async () => { @@ -141,6 +149,34 @@ test("ocr: 'hosted' sends a pdf with scanned pages to Firecrawl Parse, and nothi }) }) +test("parseMode picks the Parse pdf mode, 'auto' by default", async () => { + await withHostedStub(HOSTED, async (hits, url, sent) => { + await toMarkdown(MIXED, { ocr: 'hosted' }) + await toMarkdownBytes(await readFile(MIXED), 'pdf', { ocr: 'hosted', parseMode: 'ocr' }) + await toMarkdown(MIXED, { ocr: 'hosted', parseMode: 'fast' }) + assert.deepEqual( + sent.map((options) => options.parsers), + [[{ type: 'pdf', mode: 'auto' }], [{ type: 'pdf', mode: 'ocr' }], [{ type: 'pdf', mode: 'fast' }]], + ) + }) +}) + +test('an invalid parseMode rejects before anything is sent', async () => { + await withHostedStub(HOSTED, async (hits) => { + for (const convert of [ + () => toMarkdown(MIXED, { ocr: 'hosted', parseMode: 'OCR' }), + async () => toMarkdownBytes(await readFile(MIXED), 'pdf', { ocr: 'hosted', parseMode: 'accurate' }), + ]) { + await assert.rejects(convert(), (error) => { + assert.equal(error.code, 'hosted') + assert.match(error.message, /invalid parseMode .*expected one of: auto, fast, ocr/) + return true + }) + } + assert.deepEqual(hits, []) + }) +}) + test('the keyless limit says to set an api key', async () => { const limited = { status: 429, body: { success: false, error: 'Rate limit exceeded' } } await withHostedStub(limited, async () => { @@ -205,8 +241,24 @@ test('cli --ocr hosted converts a pdf with scanned pages through Firecrawl Parse }) }) +test('cli --parse-mode picks the Parse pdf mode', async () => { + await withHostedStub(HOSTED, async (hits, url, sent) => { + const { code } = await runCli([MIXED, '--ocr', 'hosted', '--parse-mode', 'ocr', '--api-url', url]) + assert.equal(code, undefined) + assert.deepEqual(sent[0].parsers, [{ type: 'pdf', mode: 'ocr' }]) + }) +}) + test('cli exits 2 on usage errors', async () => { - for (const args of [[], ['--frmat', 'csv', CSV], ['--format', 'nope', CSV], [OUTLINE, RICH], ['--ocr', 'cloud', MIXED]]) { + const usageErrors = [ + [], + ['--frmat', 'csv', CSV], + ['--format', 'nope', CSV], + [OUTLINE, RICH], + ['--ocr', 'cloud', MIXED], + ['--parse-mode', 'nope', MIXED], + ] + for (const args of usageErrors) { const { code, stderr } = await runCli(args) assert.equal(code, 2) assert.match(stderr, /^anydoc: /) diff --git a/python/README.md b/python/README.md index 9d038bed..9680b568 100644 --- a/python/README.md +++ b/python/README.md @@ -52,6 +52,12 @@ anydoc converts locally and does not do OCR, so a PDF with scanned or image-only markdown = anydoc.to_markdown("scan.pdf", ocr="hosted") ``` +`parse_mode` picks the Parse PDF mode: `"auto"` (the default) extracts text and falls back to OCR, `"fast"` extracts text only, `"ocr"` OCRs every page. Any other value raises `HostedError` before anything is sent. + +```python +markdown = anydoc.to_markdown("scan.pdf", ocr="hosted", parse_mode="ocr") +``` + ## Errors A conversion raises only when no complete Markdown could come out of the file. The exception type names what went wrong: diff --git a/python/anydoc/__init__.py b/python/anydoc/__init__.py index c1ba6560..1cd64e73 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc/__init__.py @@ -7,7 +7,7 @@ import uuid from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import Literal +from typing import Literal, get_args from anydoc._anydoc import ( Asset, @@ -51,6 +51,11 @@ Firecrawl Parse instead, keyless unless a key is given. Documents anydoc converts itself never leave the machine.""" +ParseMode = Literal["auto", "fast", "ocr"] +"""Firecrawl Parse PDF mode for `ocr="hosted"`. `auto` (the default) extracts +text and falls back to OCR, `fast` extracts text only, `ocr` OCRs every +page.""" + class HostedError(ConvertError): """`ocr="hosted"` could not get the document through Firecrawl Parse.""" @@ -60,6 +65,7 @@ def to_markdown( path: "str | os.PathLike[str]", *, ocr: Ocr = "reject", + parse_mode: ParseMode = "auto", api_key: "str | None" = None, api_url: "str | None" = None, ) -> str: @@ -67,16 +73,18 @@ def to_markdown( file content; the extension is the fallback for signature-less formats (CSV) and unrecognizable containers. - For `ocr="hosted"`, `api_key` falls back to `FIRECRAWL_API_KEY`, then - keyless; `api_url` to `FIRECRAWL_API_URL`, then - `https://api.firecrawl.dev`.""" + For `ocr="hosted"`, `parse_mode` picks the Parse PDF mode, and any other + value raises `HostedError` before anything is sent; `api_key` falls back + to `FIRECRAWL_API_KEY`, then keyless; `api_url` to `FIRECRAWL_API_URL`, + then `https://api.firecrawl.dev`.""" + _check_parse_mode(parse_mode) try: return _to_markdown(path) except NeedsOcrError: if ocr != "hosted": raise path = Path(path) - return _parse_hosted(path.read_bytes(), path.name, api_key, api_url) + return _parse_hosted(path.read_bytes(), path.name, parse_mode, api_key, api_url) def to_markdown_bytes( @@ -84,33 +92,43 @@ def to_markdown_bytes( format: "Format | None" = None, *, ocr: Ocr = "reject", + parse_mode: ParseMode = "auto", api_key: "str | None" = None, api_url: "str | None" = None, ) -> str: """Convert an in-memory document to Markdown. Without a format, it is detected from the content, which signature-less formats (CSV) have to - name explicitly. `ocr`, `api_key` and `api_url` are as for - `to_markdown`.""" + name explicitly. `ocr`, `parse_mode`, `api_key` and `api_url` are as + for `to_markdown`.""" + _check_parse_mode(parse_mode) try: return _to_markdown_bytes(data, format) except NeedsOcrError: if ocr != "hosted": raise - return _parse_hosted(bytes(data), "document.pdf", api_key, api_url) + return _parse_hosted(bytes(data), "document.pdf", parse_mode, api_key, api_url) _API_URL = "https://api.firecrawl.dev" _TIMEOUT_SECONDS = 300 +def _check_parse_mode(parse_mode: str) -> None: + if parse_mode not in get_args(ParseMode): + expected = ", ".join(get_args(ParseMode)) + raise HostedError(f"invalid parse_mode {parse_mode!r}; expected one of: {expected}") + + # The whole document goes, not only the pages that need OCR: Parse has no # page selection. -def _parse_hosted(data: bytes, filename: str, api_key: "str | None", api_url: "str | None") -> str: +def _parse_hosted( + data: bytes, filename: str, parse_mode: str, api_key: "str | None", api_url: "str | None" +) -> str: if api_key is None: api_key = os.environ.get("FIRECRAWL_API_KEY") api_url = api_url or os.environ.get("FIRECRAWL_API_URL") or _API_URL url = api_url.rstrip("/") + "/v2/parse" - options = {"parsers": [{"type": "pdf", "mode": "auto"}], "origin": f"anydoc@{_version()}"} + options = {"parsers": [{"type": "pdf", "mode": parse_mode}], "origin": f"anydoc@{_version()}"} boundary = uuid.uuid4().hex request = urllib.request.Request( url, @@ -200,6 +218,7 @@ def _version() -> str: "NeedsOcrError", "Note", "Ocr", + "ParseMode", "ResourceLimitError", "Style", "Table", diff --git a/python/tests/test_anydoc.py b/python/tests/test_anydoc.py index 8cff940e..73d3d6d2 100644 --- a/python/tests/test_anydoc.py +++ b/python/tests/test_anydoc.py @@ -25,16 +25,19 @@ @contextmanager -def hosted_stub(status, body): +def hosted_stub(status, body, sent=None): """A stand-in for api.firecrawl.dev that answers every request with one - reply and records each hit as (path, whether a PDF came with it). The - block runs keyless, whatever the environment.""" + reply and records each hit as (path, whether a PDF came with it), and the + Parse options each one sent into `sent`. The block runs keyless, whatever + the environment.""" hits = [] class Handler(BaseHTTPRequestHandler): def do_POST(self): payload = self.rfile.read(int(self.headers.get("content-length", 0))) hits.append((self.path, b"%PDF-" in payload)) + if sent is not None: + sent.append(parse_options(payload)) reply = json.dumps(body).encode() self.send_response(status) self.send_header("content-type", "application/json") @@ -61,6 +64,11 @@ def log_message(self, *args): os.environ[name] = value +def parse_options(payload): + options = payload.split(b'name="options"\r\n\r\n', 1)[1] + return json.loads(options.split(b"\r\n--", 1)[0]) + + class AnydocTest(unittest.TestCase): def test_to_markdown_detects_the_format_from_the_file_content(self): markdown = anydoc.to_markdown(OUTLINE) @@ -141,6 +149,32 @@ def test_ocr_hosted_sends_a_pdf_with_scanned_pages_to_firecrawl_parse_and_nothin self.assertRegex(anydoc.to_markdown(OUTLINE, ocr="hosted"), r"(?m)^# ") self.assertEqual(hits, [("/v2/parse", True)]) + def test_parse_mode_picks_the_parse_pdf_mode_auto_by_default(self): + reply = {"success": True, "data": {"markdown": HOSTED_MARKDOWN}} + sent = [] + with hosted_stub(200, reply, sent): + anydoc.to_markdown(MIXED, ocr="hosted") + anydoc.to_markdown_bytes(MIXED.read_bytes(), "pdf", ocr="hosted", parse_mode="ocr") + anydoc.to_markdown(MIXED, ocr="hosted", parse_mode="fast") + self.assertEqual( + [options["parsers"] for options in sent], + [ + [{"type": "pdf", "mode": "auto"}], + [{"type": "pdf", "mode": "ocr"}], + [{"type": "pdf", "mode": "fast"}], + ], + ) + + def test_an_invalid_parse_mode_raises_before_anything_is_sent(self): + reply = {"success": True, "data": {"markdown": HOSTED_MARKDOWN}} + expected = "invalid parse_mode .*expected one of: auto, fast, ocr" + with hosted_stub(200, reply) as hits: + with self.assertRaisesRegex(anydoc.HostedError, expected): + anydoc.to_markdown(MIXED, ocr="hosted", parse_mode="OCR") + with self.assertRaisesRegex(anydoc.HostedError, expected): + anydoc.to_markdown_bytes(MIXED.read_bytes(), ocr="hosted", parse_mode="accurate") + self.assertEqual(hits, []) + def test_the_keyless_limit_says_to_set_an_api_key(self): with hosted_stub(429, {"success": False, "error": "Rate limit exceeded"}): with self.assertRaisesRegex(anydoc.HostedError, "set FIRECRAWL_API_KEY"): @@ -162,7 +196,8 @@ def test_the_stubs_cover_the_module(self): exported = {name for name in dir(anydoc._anydoc) if not name.startswith("_")} self.assertEqual(stubbed, exported) # __init__.py re-exports the whole module, plus what it adds itself. - self.assertEqual(set(anydoc.__all__), exported | {"Format", "HostedError", "Ocr"}) + added = {"Format", "HostedError", "Ocr", "ParseMode"} + self.assertEqual(set(anydoc.__all__), exported | added) if __name__ == "__main__": From 139087f38e70ef2c941674941bf39b14b644d87e Mon Sep 17 00:00:00 2001 From: erikengervall <5392475+erikengervall@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:01:17 +0000 Subject: [PATCH 2/2] test(hosted): reject a bad parse mode on a document that converts locally --- node/test.mjs | 2 +- python/tests/test_anydoc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node/test.mjs b/node/test.mjs index c7028c9b..3636206e 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -164,7 +164,7 @@ test("parseMode picks the Parse pdf mode, 'auto' by default", async () => { test('an invalid parseMode rejects before anything is sent', async () => { await withHostedStub(HOSTED, async (hits) => { for (const convert of [ - () => toMarkdown(MIXED, { ocr: 'hosted', parseMode: 'OCR' }), + () => toMarkdown(OUTLINE, { ocr: 'hosted', parseMode: 'OCR' }), async () => toMarkdownBytes(await readFile(MIXED), 'pdf', { ocr: 'hosted', parseMode: 'accurate' }), ]) { await assert.rejects(convert(), (error) => { diff --git a/python/tests/test_anydoc.py b/python/tests/test_anydoc.py index 73d3d6d2..32a42ad6 100644 --- a/python/tests/test_anydoc.py +++ b/python/tests/test_anydoc.py @@ -170,7 +170,7 @@ def test_an_invalid_parse_mode_raises_before_anything_is_sent(self): expected = "invalid parse_mode .*expected one of: auto, fast, ocr" with hosted_stub(200, reply) as hits: with self.assertRaisesRegex(anydoc.HostedError, expected): - anydoc.to_markdown(MIXED, ocr="hosted", parse_mode="OCR") + anydoc.to_markdown(OUTLINE, ocr="hosted", parse_mode="OCR") with self.assertRaisesRegex(anydoc.HostedError, expected): anydoc.to_markdown_bytes(MIXED.read_bytes(), ocr="hosted", parse_mode="accurate") self.assertEqual(hits, [])