feat(hosted): let callers choose the Parse mode

README.md

@@ -142,7 +142,7 @@ anydoc reads text-based PDFs locally but does no OCR, so a PDF with scanned or i
 | Node   | `toMarkdown('scan.pdf', { ocr: 'hosted' })`    | `apiKey`                      |
 | Python | `anydoc.to_markdown("scan.pdf", ocr="hosted")` | `api_key`                     |
 
-Only documents that need OCR leave the machine, and the whole document goes, since Parse has no page selection. If Parse cannot convert it, Node rejects with `code: 'hosted'` and Python raises `HostedError`. `--api-url`, `apiUrl` and `api_url`, else `FIRECRAWL_API_URL`, point at another Parse deployment. The Rust crate has no `ocr` option and never makes network calls.
+Only documents that need OCR leave the machine, and the whole document goes, since Parse has no page selection. If Parse cannot convert it, Node rejects with `code: 'hosted'` and Python raises `HostedError`. `--api-url`, `apiUrl` and `api_url`, else `FIRECRAWL_API_URL`, point at another Parse deployment. `--parse-mode`, `parseMode` and `parse_mode` pick the Parse PDF mode: `auto` (the default) extracts text and falls back to OCR, `fast` extracts text only, `ocr` OCRs every page. The Rust crate has no `ocr` option and never makes network calls.
 
 ## Features
 

node/README.md

@@ -63,7 +63,13 @@ anydoc converts locally and does not do OCR, so a PDF with scanned or image-only
 const markdown = await toMarkdown('scan.pdf', { ocr: 'hosted' });
 ```
 
-On the CLI, `anydoc scan.pdf --ocr hosted`.
+`parseMode` picks the Parse PDF mode: `'auto'` (the default) extracts text and falls back to OCR, `'fast'` extracts text only, `'ocr'` OCRs every page. Any other value rejects with `code: 'hosted'` before anything is sent.
+
+```js
+const markdown = await toMarkdown('scan.pdf', { ocr: 'hosted', parseMode: 'ocr' });
+```
+
+On the CLI, `anydoc scan.pdf --ocr hosted`, and `--parse-mode ocr` to pick the mode.
 
 ## Errors
 

node/anydoc.d.ts

@@ -10,6 +10,12 @@ export interface ConvertOptions {
    * machine.
    */
   ocr?: 'reject' | 'hosted'
+  /**
+   * Firecrawl Parse PDF mode for `hosted`: `auto` (the default) extracts text
+   * and falls back to OCR, `fast` extracts text only, `ocr` OCRs every page.
+   * Any other value rejects with `code: 'hosted'` before anything is sent.
+   */
+  parseMode?: 'auto' | 'fast' | 'ocr'
   /** Firecrawl API key for `hosted`, else `FIRECRAWL_API_KEY`, else keyless. */
   apiKey?: string
   /** Firecrawl API URL for `hosted`, else `FIRECRAWL_API_URL`, else `https://api.firecrawl.dev`. */

node/anydoc.js

@@ -8,13 +8,16 @@ const { version } = require('./package.json')
 
 const API_URL = 'https://api.firecrawl.dev'
 const TIMEOUT_MS = 300_000
+const PARSE_MODES = ['auto', 'fast', 'ocr']
 
 /**
  * Convert a document file to Markdown. `options.ocr` decides what happens to
  * a PDF whose pages need OCR: `'reject'` (the default) rejects with
- * `needsOcr`, `'hosted'` sends the document to Firecrawl Parse instead.
+ * `needsOcr`, `'hosted'` sends the document to Firecrawl Parse instead, in
+ * `options.parseMode` (`'auto'` by default).
  */
 async function toMarkdown(path, options) {
+  checkParseMode(options)
   try {
     return await native.toMarkdown(path)
   } catch (error) {
@@ -25,6 +28,7 @@ async function toMarkdown(path, options) {
 
 /** `toMarkdown` for bytes; `options` as there. */
 async function toMarkdownBytes(bytes, format, options) {
+  checkParseMode(options)
   try {
     return await native.toMarkdownBytes(bytes, format)
   } catch (error) {
@@ -33,6 +37,13 @@ async function toMarkdownBytes(bytes, format, options) {
   }
 }
 
+function checkParseMode(options) {
+  const mode = options?.parseMode
+  if (mode !== undefined && !PARSE_MODES.includes(mode)) {
+    throw hostedError(`invalid parseMode '${mode}'; expected one of: ${PARSE_MODES.join(', ')}`)
+  }
+}
+
 function sendsToHosted(error, options) {
   return error.code === 'needsOcr' && options?.ocr === 'hosted'
 }
@@ -43,7 +54,7 @@ async function parseHosted(bytes, filename, options) {
   const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY
   const apiUrl = options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? API_URL
   const url = `${apiUrl.replace(/\/$/, '')}/v2/parse`
-  const parse = { parsers: [{ type: 'pdf', mode: 'auto' }], origin: `anydoc@${version}` }
+  const parse = { parsers: [{ type: 'pdf', mode: options.parseMode ?? 'auto' }], origin: `anydoc@${version}` }
   const body = new FormData()
   body.append('options', JSON.stringify(parse))
   body.append('file', new Blob([bytes], { type: 'application/pdf' }), filename)

node/cli.js

@@ -24,6 +24,9 @@ Options:
   --ocr <mode>           What to do with a PDF whose pages need OCR:
                          reject (default) exits 3; hosted sends the
                          document to Firecrawl Parse
+  --parse-mode <mode>    Firecrawl Parse PDF mode for --ocr hosted: auto
+                         (default) falls back to OCR, fast extracts text
+                         only, ocr OCRs every page
   --api-key <key>        Firecrawl API key for --ocr hosted, else
                          FIRECRAWL_API_KEY, else keyless
   --api-url <url>        Firecrawl API URL for --ocr hosted, else
@@ -52,6 +55,7 @@ Examples:
 `
 
 const OCR_MODES = ['reject', 'hosted']
+const PARSE_MODES = ['auto', 'fast', 'ocr']
 
 const USAGE_ERROR = 2
 const CONVERSION_ERROR = 1
@@ -63,7 +67,7 @@ function fail(code, message) {
 }
 
 function parseArgs(argv) {
-  const args = { input: null, output: null, format: null, ocr: null, apiKey: null, apiUrl: null }
+  const args = { input: null, output: null, format: null, ocr: null, parseMode: null, apiKey: null, apiUrl: null }
   let positionalOnly = false
   for (let i = 0; i < argv.length; i++) {
     let arg = argv[i]
@@ -114,6 +118,12 @@ function parseArgs(argv) {
           fail(USAGE_ERROR, `invalid --ocr '${args.ocr}'; expected one of: ${OCR_MODES.join(', ')}`)
         }
         break
+      case '--parse-mode':
+        args.parseMode = value()
+        if (!PARSE_MODES.includes(args.parseMode)) {
+          fail(USAGE_ERROR, `invalid --parse-mode '${args.parseMode}'; expected one of: ${PARSE_MODES.join(', ')}`)
+        }
+        break
       case '--api-key':
         args.apiKey = value()
         break
@@ -156,7 +166,12 @@ async function main() {
     }
   }
 
-  const options = { ocr: args.ocr ?? undefined, apiKey: args.apiKey ?? undefined, apiUrl: args.apiUrl ?? undefined }
+  const options = {
+    ocr: args.ocr ?? undefined,
+    parseMode: args.parseMode ?? undefined,
+    apiKey: args.apiKey ?? undefined,
+    apiUrl: args.apiUrl ?? undefined,
+  }
   let markdown
   try {
     if (args.input === '-') {

node/test.mjs

@@ -101,15 +101,18 @@ test('the package exports everything the native binding does', async () => {
 })
 
 // A stand-in for api.firecrawl.dev that answers every request with `reply`
-// and records each hit as [path, whether a PDF came with it]. Runs `run`
-// keyless, whatever the environment.
+// and records each hit as [path, whether a PDF came with it], and the Parse
+// options each one sent. Runs `run` keyless, whatever the environment.
 async function withHostedStub(reply, run) {
   const hits = []
+  const sent = []
   const server = createServer((request, response) => {
     const chunks = []
     request.on('data', (chunk) => chunks.push(chunk))
     request.on('end', () => {
-      hits.push([request.url, Buffer.concat(chunks).includes('%PDF-')])
+      const payload = Buffer.concat(chunks)
+      hits.push([request.url, payload.includes('%PDF-')])
+      sent.push(parseOptions(payload.toString('latin1')))
       response.writeHead(reply.status, { 'content-type': 'application/json' })
       response.end(JSON.stringify(reply.body))
     })
@@ -120,7 +123,7 @@ async function withHostedStub(reply, run) {
   process.env.FIRECRAWL_API_URL = url
   delete process.env.FIRECRAWL_API_KEY
   try {
-    await run(hits, url)
+    await run(hits, url, sent)
   } finally {
     for (const [name, value] of Object.entries(saved)) {
       if (value === undefined) delete process.env[name]
@@ -130,6 +133,11 @@ async function withHostedStub(reply, run) {
   }
 }
 
+function parseOptions(payload) {
+  const start = payload.indexOf('name="options"\r\n\r\n') + 'name="options"\r\n\r\n'.length
+  return JSON.parse(payload.slice(start, payload.indexOf('\r\n--', start)))
+}
+
 const HOSTED = { status: 200, body: { success: true, data: { markdown: '# Read by the hosted parser\n' } } }
 
 test("ocr: 'hosted' sends a pdf with scanned pages to Firecrawl Parse, and nothing else", async () => {
@@ -141,6 +149,34 @@ test("ocr: 'hosted' sends a pdf with scanned pages to Firecrawl Parse, and nothi
   })
 })
 
+test("parseMode picks the Parse pdf mode, 'auto' by default", async () => {
+  await withHostedStub(HOSTED, async (hits, url, sent) => {
+    await toMarkdown(MIXED, { ocr: 'hosted' })
+    await toMarkdownBytes(await readFile(MIXED), 'pdf', { ocr: 'hosted', parseMode: 'ocr' })
+    await toMarkdown(MIXED, { ocr: 'hosted', parseMode: 'fast' })
+    assert.deepEqual(
+      sent.map((options) => options.parsers),
+      [[{ type: 'pdf', mode: 'auto' }], [{ type: 'pdf', mode: 'ocr' }], [{ type: 'pdf', mode: 'fast' }]],
+    )
+  })
+})
+
+test('an invalid parseMode rejects before anything is sent', async () => {
+  await withHostedStub(HOSTED, async (hits) => {
+    for (const convert of [
+      () => toMarkdown(OUTLINE, { ocr: 'hosted', parseMode: 'OCR' }),
+      async () => toMarkdownBytes(await readFile(MIXED), 'pdf', { ocr: 'hosted', parseMode: 'accurate' }),
+    ]) {
+      await assert.rejects(convert(), (error) => {
+        assert.equal(error.code, 'hosted')
+        assert.match(error.message, /invalid parseMode .*expected one of: auto, fast, ocr/)
+        return true
+      })
+    }
+    assert.deepEqual(hits, [])
+  })
+})
+
 test('the keyless limit says to set an api key', async () => {
   const limited = { status: 429, body: { success: false, error: 'Rate limit exceeded' } }
   await withHostedStub(limited, async () => {
@@ -205,8 +241,24 @@ test('cli --ocr hosted converts a pdf with scanned pages through Firecrawl Parse
   })
 })
 
+test('cli --parse-mode picks the Parse pdf mode', async () => {
+  await withHostedStub(HOSTED, async (hits, url, sent) => {
+    const { code } = await runCli([MIXED, '--ocr', 'hosted', '--parse-mode', 'ocr', '--api-url', url])
+    assert.equal(code, undefined)
+    assert.deepEqual(sent[0].parsers, [{ type: 'pdf', mode: 'ocr' }])
+  })
+})
+
 test('cli exits 2 on usage errors', async () => {
-  for (const args of [[], ['--frmat', 'csv', CSV], ['--format', 'nope', CSV], [OUTLINE, RICH], ['--ocr', 'cloud', MIXED]]) {
+  const usageErrors = [
+    [],
+    ['--frmat', 'csv', CSV],
+    ['--format', 'nope', CSV],
+    [OUTLINE, RICH],
+    ['--ocr', 'cloud', MIXED],
+    ['--parse-mode', 'nope', MIXED],
+  ]
+  for (const args of usageErrors) {
     const { code, stderr } = await runCli(args)
     assert.equal(code, 2)
     assert.match(stderr, /^anydoc: /)

python/README.md

@@ -52,6 +52,12 @@ anydoc converts locally and does not do OCR, so a PDF with scanned or image-only
 markdown = anydoc.to_markdown("scan.pdf", ocr="hosted")
 ```
 
+`parse_mode` picks the Parse PDF mode: `"auto"` (the default) extracts text and falls back to OCR, `"fast"` extracts text only, `"ocr"` OCRs every page. Any other value raises `HostedError` before anything is sent.
+
+```python
+markdown = anydoc.to_markdown("scan.pdf", ocr="hosted", parse_mode="ocr")
+```
+
 ## Errors
 
 A conversion raises only when no complete Markdown could come out of the file. The exception type names what went wrong:

python/anydoc/__init__.py

@@ -7,7 +7,7 @@
 import uuid
 from importlib.metadata import PackageNotFoundError, version
 from pathlib import Path
-from typing import Literal
+from typing import Literal, get_args
 
 from anydoc._anydoc import (
     Asset,
@@ -51,6 +51,11 @@
 Firecrawl Parse instead, keyless unless a key is given. Documents anydoc
 converts itself never leave the machine."""
 
+ParseMode = Literal["auto", "fast", "ocr"]
+"""Firecrawl Parse PDF mode for `ocr="hosted"`. `auto` (the default) extracts
+text and falls back to OCR, `fast` extracts text only, `ocr` OCRs every
+page."""
+
 
 class HostedError(ConvertError):
     """`ocr="hosted"` could not get the document through Firecrawl Parse."""
@@ -60,57 +65,70 @@ def to_markdown(
     path: "str | os.PathLike[str]",
     *,
     ocr: Ocr = "reject",
+    parse_mode: ParseMode = "auto",
     api_key: "str | None" = None,
     api_url: "str | None" = None,
 ) -> str:
     """Convert a document file to Markdown. The format is detected from the
     file content; the extension is the fallback for signature-less formats
     (CSV) and unrecognizable containers.
 
-    For `ocr="hosted"`, `api_key` falls back to `FIRECRAWL_API_KEY`, then
-    keyless; `api_url` to `FIRECRAWL_API_URL`, then
-    `https://api.firecrawl.dev`."""
+    For `ocr="hosted"`, `parse_mode` picks the Parse PDF mode, and any other
+    value raises `HostedError` before anything is sent; `api_key` falls back
+    to `FIRECRAWL_API_KEY`, then keyless; `api_url` to `FIRECRAWL_API_URL`,
+    then `https://api.firecrawl.dev`."""
+    _check_parse_mode(parse_mode)
     try:
         return _to_markdown(path)
     except NeedsOcrError:
         if ocr != "hosted":
             raise
     path = Path(path)
-    return _parse_hosted(path.read_bytes(), path.name, api_key, api_url)
+    return _parse_hosted(path.read_bytes(), path.name, parse_mode, api_key, api_url)
 
 
 def to_markdown_bytes(
     data: "bytes | bytearray",
     format: "Format | None" = None,
     *,
     ocr: Ocr = "reject",
+    parse_mode: ParseMode = "auto",
     api_key: "str | None" = None,
     api_url: "str | None" = None,
 ) -> str:
     """Convert an in-memory document to Markdown. Without a format, it is
     detected from the content, which signature-less formats (CSV) have to
-    name explicitly. `ocr`, `api_key` and `api_url` are as for
-    `to_markdown`."""
+    name explicitly. `ocr`, `parse_mode`, `api_key` and `api_url` are as
+    for `to_markdown`."""
+    _check_parse_mode(parse_mode)
     try:
         return _to_markdown_bytes(data, format)
     except NeedsOcrError:
         if ocr != "hosted":
             raise
-    return _parse_hosted(bytes(data), "document.pdf", api_key, api_url)
+    return _parse_hosted(bytes(data), "document.pdf", parse_mode, api_key, api_url)
 
 
 _API_URL = "https://api.firecrawl.dev"
 _TIMEOUT_SECONDS = 300
 
 
+def _check_parse_mode(parse_mode: str) -> None:
+    if parse_mode not in get_args(ParseMode):
+        expected = ", ".join(get_args(ParseMode))
+        raise HostedError(f"invalid parse_mode {parse_mode!r}; expected one of: {expected}")
+
+
 # The whole document goes, not only the pages that need OCR: Parse has no
 # page selection.
-def _parse_hosted(data: bytes, filename: str, api_key: "str | None", api_url: "str | None") -> str:
+def _parse_hosted(
+    data: bytes, filename: str, parse_mode: str, api_key: "str | None", api_url: "str | None"
+) -> str:
     if api_key is None:
         api_key = os.environ.get("FIRECRAWL_API_KEY")
     api_url = api_url or os.environ.get("FIRECRAWL_API_URL") or _API_URL
     url = api_url.rstrip("/") + "/v2/parse"
-    options = {"parsers": [{"type": "pdf", "mode": "auto"}], "origin": f"anydoc@{_version()}"}
+    options = {"parsers": [{"type": "pdf", "mode": parse_mode}], "origin": f"anydoc@{_version()}"}
     boundary = uuid.uuid4().hex
     request = urllib.request.Request(
         url,
@@ -200,6 +218,7 @@ def _version() -> str:
     "NeedsOcrError",
     "Note",
     "Ocr",
+    "ParseMode",
     "ResourceLimitError",
     "Style",
     "Table",