From 822d6d3ecbf9e29d40d5e20c62deca2b5c08ba76 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Sat, 12 Sep 2026 14:26:39 +0100 Subject: [PATCH 1/2] Pin Falcon-OCR to v1 and add a falcon-ocr-1.5 config tiiuae/Falcon-OCR now holds two model versions in one repo: TII overwrote main with the v1.5 weights on 2026-09-11 without a tag or branch. The `falcon-ocr` slug loaded main, so it silently became v1.5 and stopped matching the v1 rows already judged. - `falcon-ocr` pins the last v1 commit via `--revision`. - New `falcon-ocr-1.5` slug pins the v1.5 release head. Same model_id; the dataset loader already relabels colliding configs by slug so both stay distinct leaderboard rows. - Tests: registry count 17; both slugs must pin distinct full shas. Depends on the `--revision` flag landing in uv-scripts/ocr falcon-ocr.py (davanstrien/uv-scripts-for-ai, branch ocr/falcon-ocr-revision) and the Hub mirror syncing. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TWTqZnz7cp1TzvicRo6FkN --- README.md | 2 +- src/ocr_bench/run.py | 23 ++++++++++++++++++++++- tests/test_run.py | 18 ++++++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 29d1585..de1901e 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ ocr-bench ships with 16 registered OCR models. Five form the default set: | `deepseek-ocr` | 4B | Diverse documents | Most consistent across types | | `dots-ocr` | 1.7B | General | Struggles on historical text | -The opt-in registry also includes compact models at different quality/cost points, including `falcon-ocr` (0.3B) and `ovis-ocr2` (0.9B). List every available model with: +The opt-in registry also includes compact models at different quality/cost points, including `falcon-ocr` (0.3B, v1 weights), `falcon-ocr-1.5` (0.3B, the v1.5 weights TII pushed to the same repo on 2026-09-11; both slugs pin a commit) and `ovis-ocr2` (0.9B). List every available model with: ```bash ocr-bench run --list-models diff --git a/src/ocr_bench/run.py b/src/ocr_bench/run.py index 88e33fe..48c35de 100644 --- a/src/ocr_bench/run.py +++ b/src/ocr_bench/run.py @@ -43,12 +43,33 @@ class ModelConfig: _VLLM_OPENAI_ENV = {"PYTHONPATH": "/usr/local/lib/python3.12/dist-packages"} +# tiiuae/Falcon-OCR ships v1 and v1.5 in the SAME repo: TII overwrote `main` +# with the v1.5 weights on 2026-09-11 (no tag, no branch). Both entries pin a +# commit so a run reproduces regardless of where `main` points next. The +# script's `--revision` flag resolves the commit and records it in inference_info. +_FALCON_OCR_SCRIPT = "https://huggingface.co/datasets/uv-scripts/ocr/raw/main/falcon-ocr.py" +FALCON_OCR_V1_REVISION = "42ec56b72a23984ac059e7c8a6d397a8529423fe" # last v1, 2026-07-03 +FALCON_OCR_V15_REVISION = "fe757d59ecd79d4d68760162306a70a015761ad9" # v1.5 head, 2026-09-11 + + MODEL_REGISTRY: dict[str, ModelConfig] = { "falcon-ocr": ModelConfig( - script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/falcon-ocr.py", + script=_FALCON_OCR_SCRIPT, + model_id="tiiuae/Falcon-OCR", + size="0.3B", + default_flavor="l4x1", + # v1 weights. Without the pin this slug silently became v1.5 on 2026-09-11. + default_args=["--revision", FALCON_OCR_V1_REVISION], + ), + "falcon-ocr-1.5": ModelConfig( + script=_FALCON_OCR_SCRIPT, model_id="tiiuae/Falcon-OCR", size="0.3B", default_flavor="l4x1", + # v1.5 weights (same repo, same 0.3B arch). Shares model_id with + # `falcon-ocr`; the loader labels colliding configs by slug so both rows + # stay distinct on the leaderboard. + default_args=["--revision", FALCON_OCR_V15_REVISION], ), "glm-ocr": ModelConfig( script="https://huggingface.co/datasets/uv-scripts/ocr/raw/main/glm-ocr.py", diff --git a/tests/test_run.py b/tests/test_run.py index e0f6211..1228cca 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -49,7 +49,7 @@ def test_image_mode_fields_default_none(self): class TestModelRegistry: def test_has_core_models(self): - assert len(MODEL_REGISTRY) == 16 + assert len(MODEL_REGISTRY) == 17 def test_default_models_exist_in_registry(self): for slug in DEFAULT_MODELS: @@ -70,6 +70,7 @@ def test_deepseek_has_prompt_mode_free(self): def test_compact_models_use_standard_l4_launch(self): expected = { "falcon-ocr": ("tiiuae/Falcon-OCR", "0.3B", "falcon-ocr.py"), + "falcon-ocr-1.5": ("tiiuae/Falcon-OCR", "0.3B", "falcon-ocr.py"), "ovis-ocr2": ("ATH-MaaS/OvisOCR2", "0.9B", "ovis-ocr2.py"), } for slug, (model_id, size, script_name) in expected.items(): @@ -78,13 +79,26 @@ def test_compact_models_use_standard_l4_launch(self): assert cfg.size == size assert cfg.script.endswith(f"/{script_name}") assert cfg.default_flavor == "l4x1" - assert cfg.default_args == [] assert cfg.image is None assert cfg.python is None assert cfg.env is None + assert MODEL_REGISTRY["ovis-ocr2"].default_args == [] + + def test_falcon_ocr_versions_pin_distinct_commits(self): + # Same Hub repo holds v1 and v1.5 (main was overwritten 2026-09-11), so + # each slug must pin its own full commit sha or runs stop reproducing. + pins = {} + for slug in ("falcon-ocr", "falcon-ocr-1.5"): + args = MODEL_REGISTRY[slug].default_args + assert args[0] == "--revision", slug + sha = args[1] + assert len(sha) == 40 and all(c in "0123456789abcdef" for c in sha), slug + pins[slug] = sha + assert pins["falcon-ocr"] != pins["falcon-ocr-1.5"] def test_compact_models_are_opt_in(self): assert "falcon-ocr" not in DEFAULT_MODELS + assert "falcon-ocr-1.5" not in DEFAULT_MODELS assert "ovis-ocr2" not in DEFAULT_MODELS def test_image_mode_models_configured(self): From ed609c64ee23e909e13b9128d9718e074a554df5 Mon Sep 17 00:00:00 2001 From: Daniel van Strien Date: Sat, 12 Sep 2026 14:46:11 +0100 Subject: [PATCH 2/2] Point falcon-ocr-1.5 at the split falcon-ocr-1.5.py recipe uv-scripts split the Falcon recipe by version identity (falcon-ocr.py pins v1, falcon-ocr-1.5.py pins the v1.5 release head). The slug now loads the matching script. The explicit `--revision` pins stay as belt-and-braces so the bench never depends on a script default. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TWTqZnz7cp1TzvicRo6FkN --- src/ocr_bench/run.py | 11 +++++++---- tests/test_run.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ocr_bench/run.py b/src/ocr_bench/run.py index 48c35de..990e6cd 100644 --- a/src/ocr_bench/run.py +++ b/src/ocr_bench/run.py @@ -44,10 +44,13 @@ class ModelConfig: # tiiuae/Falcon-OCR ships v1 and v1.5 in the SAME repo: TII overwrote `main` -# with the v1.5 weights on 2026-09-11 (no tag, no branch). Both entries pin a -# commit so a run reproduces regardless of where `main` points next. The -# script's `--revision` flag resolves the commit and records it in inference_info. +# with the v1.5 weights on 2026-09-11 (no tag, no branch). uv-scripts split the +# recipe by version identity (falcon-ocr.py pins v1, falcon-ocr-1.5.py pins the +# v1.5 release head) and both scripts default to their pin; the explicit +# `--revision` here is belt-and-braces so the bench never depends on a script +# default. The script resolves the commit and records it in inference_info. _FALCON_OCR_SCRIPT = "https://huggingface.co/datasets/uv-scripts/ocr/raw/main/falcon-ocr.py" +_FALCON_OCR_15_SCRIPT = "https://huggingface.co/datasets/uv-scripts/ocr/raw/main/falcon-ocr-1.5.py" FALCON_OCR_V1_REVISION = "42ec56b72a23984ac059e7c8a6d397a8529423fe" # last v1, 2026-07-03 FALCON_OCR_V15_REVISION = "fe757d59ecd79d4d68760162306a70a015761ad9" # v1.5 head, 2026-09-11 @@ -62,7 +65,7 @@ class ModelConfig: default_args=["--revision", FALCON_OCR_V1_REVISION], ), "falcon-ocr-1.5": ModelConfig( - script=_FALCON_OCR_SCRIPT, + script=_FALCON_OCR_15_SCRIPT, model_id="tiiuae/Falcon-OCR", size="0.3B", default_flavor="l4x1", diff --git a/tests/test_run.py b/tests/test_run.py index 1228cca..20c7f47 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -70,7 +70,7 @@ def test_deepseek_has_prompt_mode_free(self): def test_compact_models_use_standard_l4_launch(self): expected = { "falcon-ocr": ("tiiuae/Falcon-OCR", "0.3B", "falcon-ocr.py"), - "falcon-ocr-1.5": ("tiiuae/Falcon-OCR", "0.3B", "falcon-ocr.py"), + "falcon-ocr-1.5": ("tiiuae/Falcon-OCR", "0.3B", "falcon-ocr-1.5.py"), "ovis-ocr2": ("ATH-MaaS/OvisOCR2", "0.9B", "ovis-ocr2.py"), } for slug, (model_id, size, script_name) in expected.items():