diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 783a5801..8274db3c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -9,6 +9,21 @@ on: branches: [main, master] jobs: + base-install: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install base package without acceleration or development extras + run: python -m pip install . + - name: Verify installed extraction without pandas or SciPy + run: python -m benchmark.issue148_scipy_free --installed + test: runs-on: ${{ matrix.os }} env: @@ -26,8 +41,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e .[yaml,mcp] - pip install pytest pytest-cov pytest-mock + pip install -e .[yaml,mcp,fast] + pip install pytest pytest-cov pytest-mock pandas - name: Run tests (non-COM suite) if: runner.os != 'Windows' run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index bfb4c19a..c316011e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,32 @@ All notable changes to this project are documented in this file. This changelog follows the [Keep a Changelog](https://keepachangelog.com/) format and covers changes from v0.2.70 onward. -## [Unreleased] +## [Unreleased] + +### Changed + +- Replaced production pandas cell extraction with direct openpyxl worksheet + iteration and lazy xlrd reading for legacy `.xls` files, preserving existing + cell normalization and serialized output. +- Shared extraction-scoped openpyxl workbooks across compatible stages and + per-sheet table detection, opening the formula variant only when requested. +- Isolated COM and rendering imports from light extraction while retaining + best-effort OOXML shapes/charts and legacy helper override behavior. +- Moved SciPy to the optional `fast` extra (also included in `all`), retaining + accelerated selection when installed and the existing Python fallback otherwise. + +### Added + +- Added regression coverage for previous-reader parity, workbook resource + lifetime and loader counts, subprocess import boundaries, and clustering + equivalence with SciPy-free execution. +- Added a reproducible comparison of SciPy, Python and sparse-set border + clustering, with exact output equivalence and recorded Windows timings. + +### Fixed + +- Preserved live pipeline backend overrides through lazy backend resolution. +- Avoided materializing blank cells when reading a shared sparse worksheet. ## [0.8.2] - 2026-09-26 diff --git a/README.ja.md b/README.ja.md index 2c501015..d3daf3e0 100644 --- a/README.ja.md +++ b/README.ja.md @@ -60,9 +60,10 @@ pip install exstruct オプション依存: - YAML: `pip install pyyaml` +- 罫線クラスタリングの高速化: `pip install exstruct[fast]`（SciPy）。未導入時は既存の Python 実装へフォールバックします。NumPy は引き続き必須依存です。 - TOON: `pip install python-toon` - レンダリング（PDF/PNG）: Excel + `pip install pypdfium2 pillow`（`mode=libreoffice` では非対応） -- まとめて導入: `pip install exstruct[yaml,toon,render]` +- まとめて導入: `pip install exstruct[yaml,toon,render,fast]` プラットフォーム注意: diff --git a/README.md b/README.md index 16023e4e..78e0f520 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,13 @@ LLM/RAG pipelines and local automation. pip install exstruct ``` -Optional extras: - +Optional extras: + +- Border-clustering acceleration: `pip install exstruct[fast]` (SciPy). Base extraction uses the existing Python fallback when SciPy is unavailable; NumPy remains a core dependency. - YAML: `pip install pyyaml` - TOON: `pip install python-toon` - Rendering (PDF/PNG): Excel + `pip install pypdfium2 pillow` (`mode=libreoffice` is not supported) -- Install everything at once: `pip install exstruct[yaml,toon,render]` +- Install these extras at once: `pip install exstruct[yaml,toon,render,fast]` Platform note: diff --git a/benchmark/baselines/issue148-2026-10-03-correctness.json b/benchmark/baselines/issue148-2026-10-03-correctness.json new file mode 100644 index 00000000..77c6cc62 --- /dev/null +++ b/benchmark/baselines/issue148-2026-10-03-correctness.json @@ -0,0 +1,124 @@ +{ + "schema_version": 1, + "timestamp_utc": "2026-10-03T05:40:38.779016+00:00", + "source_revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "source_dirty": true, + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "numpy": "2.4.6", + "scipy": "1.17.1", + "pandas": "3.0.6", + "openpyxl": "3.1.5" + }, + "environment": {}, + "mode": "light", + "warm_repeats": 0, + "check_only": true, + "cases": { + "sparse-small": { + "shape": [ + 40, + 20 + ], + "occupied": 44, + "grid_bytes": 800, + "grid_sha256": "0cd619ad2bd7fdf774c43d425d46517f184fc920f564310d1bdbf54db4b72e19", + "fixture_sha256": "bd45bc2a80e9d05e01ce2d92b9ed4d0044334bbb43bc0d4908e16727044ee0c8", + "min_size": 4, + "cluster": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "light_extraction_serialization": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "components": 2, + "output_sha256": "445a94b9270a585d90837a5300c1b69b991dd2dfacc39c4372ca602b3c0dfc64", + "output_bytes": 949, + "bbox_order_equal": true, + "serialized_output_equal": true + }, + "sparse-large": { + "shape": [ + 2000, + 200 + ], + "occupied": 74, + "grid_bytes": 400000, + "grid_sha256": "89f650b3baefdf79bcd1f302e11b3c220e3bf418b721e25524e2abb2a4efb41f", + "fixture_sha256": "2a0c9f8b89ec25861017a8b6b81fb65f401972b9fa92a523b2b015325a02098e", + "min_size": 4, + "cluster": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "light_extraction_serialization": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "components": 2, + "output_sha256": "395c4470fa9b7fc1d1b718da9bed660289418ffbeafe03410cdf3424bb7b3957", + "output_bytes": 1649, + "bbox_order_equal": true, + "serialized_output_equal": true + }, + "dense-style-heavy": { + "shape": [ + 300, + 80 + ], + "occupied": 24000, + "grid_bytes": 24000, + "grid_sha256": "9bf7697f1260d5f1781eb4878282d81007950425b289f00ffbea23d2b83c0ee8", + "fixture_sha256": "c6980bfa1f606bc3caad0c8c0e1204f57382e903fcacf479595c747c29b25072", + "min_size": 4, + "cluster": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "light_extraction_serialization": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "components": 1, + "output_sha256": "4027f972fe8832c95d84355c866b8235401a4453286019fa04a7a4880fdac503", + "output_bytes": 398897, + "bbox_order_equal": true, + "serialized_output_equal": true + }, + "many-disconnected": { + "shape": [ + 300, + 100 + ], + "occupied": 4000, + "grid_bytes": 30000, + "grid_sha256": "4daa7f0eafcaa14efcdd9b8b9465d5011bbdc0925e5c6da6cf69d36087d5100f", + "fixture_sha256": "a73151afb9ae566ec8908d46473693a1edf6b0449a445072df21b2a021a6bac3", + "min_size": 4, + "cluster": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "light_extraction_serialization": { + "scipy": {}, + "python": {}, + "candidate_sparse": {} + }, + "components": 1000, + "output_sha256": "ea1cff42a0a4b84b8166f36bd8e88c3db4fd0329431f56c5c32820da916e7a1a", + "output_bytes": 81361, + "bbox_order_equal": true, + "serialized_output_equal": true + } + } +} diff --git a/benchmark/baselines/issue148-2026-10-03-windows.json b/benchmark/baselines/issue148-2026-10-03-windows.json new file mode 100644 index 00000000..303c8c7c --- /dev/null +++ b/benchmark/baselines/issue148-2026-10-03-windows.json @@ -0,0 +1,388 @@ +{ + "schema_version": 1, + "timestamp_utc": "2026-10-03T05:48:54.914695+00:00", + "source_revision": "49f2d66a802edb2ceb459eddef635447cd32daec", + "source_dirty": true, + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "numpy": "2.4.6", + "scipy": "1.17.1", + "pandas": "3.0.6", + "openpyxl": "3.1.5" + }, + "environment": {}, + "mode": "light", + "warm_repeats": 7, + "check_only": false, + "cases": { + "sparse-small": { + "shape": [ + 40, + 20 + ], + "occupied": 44, + "grid_bytes": 800, + "grid_sha256": "0cd619ad2bd7fdf774c43d425d46517f184fc920f564310d1bdbf54db4b72e19", + "fixture_sha256": "bd45bc2a80e9d05e01ce2d92b9ed4d0044334bbb43bc0d4908e16727044ee0c8", + "min_size": 4, + "cluster": { + "scipy": { + "warm_ms": [ + 0.04749977961182594, + 0.03890017978847027, + 0.03300001844763756, + 0.029399991035461426, + 0.027999980375170708, + 0.02759997732937336, + 0.03060000017285347 + ], + "median_ms": 0.03060000017285347 + }, + "python": { + "warm_ms": [ + 0.07999991066753864, + 0.07700012065470219, + 0.07619988173246384, + 0.07600011304020882, + 0.07619988173246384, + 0.07690000347793102, + 0.07910002022981644 + ], + "median_ms": 0.07690000347793102 + }, + "candidate_sparse": { + "warm_ms": [ + 0.03529991954565048, + 0.031300121918320656, + 0.030299881473183632, + 0.0295999925583601, + 0.029799994081258774, + 0.031299889087677, + 0.031000003218650818 + ], + "median_ms": 0.031000003218650818 + } + }, + "light_extraction_serialization": { + "scipy": { + "warm_ms": [ + 8.671300020068884, + 8.92850011587143, + 8.848000084981322, + 8.78129992634058, + 9.132400155067444, + 8.255799999460578, + 10.280800051987171 + ], + "median_ms": 8.848000084981322 + }, + "python": { + "warm_ms": [ + 9.35940002091229, + 9.797499980777502, + 9.052100125700235, + 9.655499830842018, + 9.747199947014451, + 9.5875000115484, + 8.380299899727106 + ], + "median_ms": 9.5875000115484 + }, + "candidate_sparse": { + "warm_ms": [ + 8.6536998860538, + 7.857699878513813, + 8.204699959605932, + 7.904599886387587, + 8.510100189596415, + 9.04519995674491, + 8.986899862065911 + ], + "median_ms": 8.510100189596415 + } + }, + "components": 2, + "output_sha256": "445a94b9270a585d90837a5300c1b69b991dd2dfacc39c4372ca602b3c0dfc64", + "output_bytes": 949, + "bbox_order_equal": true, + "serialized_output_equal": true + }, + "sparse-large": { + "shape": [ + 2000, + 200 + ], + "occupied": 74, + "grid_bytes": 400000, + "grid_sha256": "89f650b3baefdf79bcd1f302e11b3c220e3bf418b721e25524e2abb2a4efb41f", + "fixture_sha256": "2a0c9f8b89ec25861017a8b6b81fb65f401972b9fa92a523b2b015325a02098e", + "min_size": 4, + "cluster": { + "scipy": { + "warm_ms": [ + 2.6639001443982124, + 2.5686000008136034, + 2.368299989029765, + 2.5859999004751444, + 2.3336000740528107, + 2.2911999840289354, + 2.5106000248342752 + ], + "median_ms": 2.5106000248342752 + }, + "python": { + "warm_ms": [ + 18.315300112590194, + 19.20010009780526, + 18.658400047570467, + 18.688400043174624, + 18.701700028032064, + 18.440400017425418, + 18.2994999922812 + ], + "median_ms": 18.658400047570467 + }, + "candidate_sparse": { + "warm_ms": [ + 0.6295999046415091, + 0.6379999686032534, + 0.6180999334901571, + 0.6071000825613737, + 0.6017000414431095, + 0.6041999440640211, + 0.6738000083714724 + ], + "median_ms": 0.6180999334901571 + } + }, + "light_extraction_serialization": { + "scipy": { + "warm_ms": [ + 1192.1276000794023, + 1223.1985998805612, + 1154.776400187984, + 1150.0510999467224, + 995.5201998818666, + 1017.136299982667, + 1128.1137999612838 + ], + "median_ms": 1150.0510999467224 + }, + "python": { + "warm_ms": [ + 1116.0206999629736, + 976.5997000504285, + 1095.5334000755101, + 1052.9799999203533, + 1109.9884998984635, + 939.1526998952031, + 1076.1782000772655 + ], + "median_ms": 1076.1782000772655 + }, + "candidate_sparse": { + "warm_ms": [ + 1049.5190001092851, + 1098.7000002060086, + 1233.6267998907715, + 1026.519700186327, + 1115.5872999224812, + 1041.2264000624418, + 1166.81810002774 + ], + "median_ms": 1098.7000002060086 + } + }, + "components": 2, + "output_sha256": "395c4470fa9b7fc1d1b718da9bed660289418ffbeafe03410cdf3424bb7b3957", + "output_bytes": 1649, + "bbox_order_equal": true, + "serialized_output_equal": true + }, + "dense-style-heavy": { + "shape": [ + 300, + 80 + ], + "occupied": 24000, + "grid_bytes": 24000, + "grid_sha256": "9bf7697f1260d5f1781eb4878282d81007950425b289f00ffbea23d2b83c0ee8", + "fixture_sha256": "c6980bfa1f606bc3caad0c8c0e1204f57382e903fcacf479595c747c29b25072", + "min_size": 4, + "cluster": { + "scipy": { + "warm_ms": [ + 0.2235998399555683, + 0.20739994943141937, + 0.21070009097456932, + 0.20599993877112865, + 0.24030008353292942, + 0.21650013513863087, + 0.305800000205636 + ], + "median_ms": 0.21650013513863087 + }, + "python": { + "warm_ms": [ + 19.95870005339384, + 19.713199930265546, + 19.830199889838696, + 19.980400102213025, + 19.562599947676063, + 20.4627001658082, + 21.421900019049644 + ], + "median_ms": 19.95870005339384 + }, + "candidate_sparse": { + "warm_ms": [ + 27.995700016617775, + 24.914200184866786, + 22.798100020736456, + 24.670499842613935, + 27.611899888142943, + 26.99670009315014, + 34.830200020223856 + ], + "median_ms": 26.99670009315014 + } + }, + "light_extraction_serialization": { + "scipy": { + "warm_ms": [ + 663.1261000875384, + 541.6727000847459, + 510.8400001190603, + 495.1317000668496, + 529.1062998585403, + 519.2692000418901, + 577.6856997981668 + ], + "median_ms": 529.1062998585403 + }, + "python": { + "warm_ms": [ + 589.8221000097692, + 597.7132001426071, + 543.329699896276, + 555.0023000687361, + 564.3450999632478, + 674.4460999034345, + 593.0543998256326 + ], + "median_ms": 589.8221000097692 + }, + "candidate_sparse": { + "warm_ms": [ + 514.3154000397772, + 506.8190998863429, + 524.3682998698205, + 533.2794000860304, + 522.8791001718491, + 534.8793000448495, + 551.3083999976516 + ], + "median_ms": 524.3682998698205 + } + }, + "components": 1, + "output_sha256": "4027f972fe8832c95d84355c866b8235401a4453286019fa04a7a4880fdac503", + "output_bytes": 398897, + "bbox_order_equal": true, + "serialized_output_equal": true + }, + "many-disconnected": { + "shape": [ + 300, + 100 + ], + "occupied": 4000, + "grid_bytes": 30000, + "grid_sha256": "4daa7f0eafcaa14efcdd9b8b9465d5011bbdc0925e5c6da6cf69d36087d5100f", + "fixture_sha256": "a73151afb9ae566ec8908d46473693a1edf6b0449a445072df21b2a021a6bac3", + "min_size": 4, + "cluster": { + "scipy": { + "warm_ms": [ + 53.962500067427754, + 52.774300100281835, + 52.7685999404639, + 53.11280000023544, + 53.15829999744892, + 54.54539996571839, + 52.32569994404912 + ], + "median_ms": 53.11280000023544 + }, + "python": { + "warm_ms": [ + 5.180099979043007, + 4.876999882981181, + 4.811800085008144, + 5.1050998736172915, + 4.875800106674433, + 4.915900062769651, + 4.806299926713109 + ], + "median_ms": 4.876999882981181 + }, + "candidate_sparse": { + "warm_ms": [ + 3.051100065931678, + 3.1623998656868935, + 3.043500008061528, + 3.146700095385313, + 2.9175998643040657, + 3.0490001663565636, + 2.908000024035573 + ], + "median_ms": 3.0490001663565636 + } + }, + "light_extraction_serialization": { + "scipy": { + "warm_ms": [ + 367.9691997822374, + 430.48650003038347, + 429.49629994109273, + 350.39399983361363, + 383.8579999282956, + 367.2038000077009, + 373.9878002088517 + ], + "median_ms": 373.9878002088517 + }, + "python": { + "warm_ms": [ + 299.3171999696642, + 298.3971000649035, + 305.6983998976648, + 317.4912000540644, + 311.9584999512881, + 296.32259998470545, + 299.0051999222487 + ], + "median_ms": 299.3171999696642 + }, + "candidate_sparse": { + "warm_ms": [ + 309.4559998717159, + 321.7763998545706, + 286.49520012550056, + 328.7179998587817, + 328.0361001379788, + 326.7642999999225, + 298.1820998247713 + ], + "median_ms": 321.7763998545706 + } + }, + "components": 1000, + "output_sha256": "ea1cff42a0a4b84b8166f36bd8e88c3db4fd0329431f56c5c32820da916e7a1a", + "output_bytes": 81361, + "bbox_order_equal": true, + "serialized_output_equal": true + } + } +} diff --git a/benchmark/baselines/issues145-148-2026-10-03-windows.json b/benchmark/baselines/issues145-148-2026-10-03-windows.json new file mode 100644 index 00000000..59971716 --- /dev/null +++ b/benchmark/baselines/issues145-148-2026-10-03-windows.json @@ -0,0 +1,3126 @@ +{ + "schema_version": 1, + "comparison_method": "Same locked interpreter; serial source-specific subprocesses; before observations retained from pre-integration pass, final after observations rerun after sparse-reader refinement; one warmup and three repeats. Exact serialized output compared independently for both modes.", + "results": [ + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:54:15.145448+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "large.xlsx", + "sha256": "72ff66fa3abde812e9ae3261530b1dd5b88c4dfd5173715b383d896b990f1b97", + "size_bytes": 185797 + }, + "mode": "light", + "startup": { + "python_process_ms": 44.35620014555752, + "package_import_process_ms": 84.95740010403097, + "cli_help_process_ms": 94.20319995842874, + "cold_extraction_process_ms": 3223.76889991574, + "cold_extraction_in_process_ms": 2490.329299820587, + "measurement_worker_process_ms": 10744.135599816218 + }, + "import_ms": 11.959400027990341, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3288.086400134489, + "serialize_ms": 78.78189999610186, + "total_ms": 3366.868300130591 + }, + "repeated": [ + { + "extract_ms": 1473.7426999490708, + "serialize_ms": 72.44429993443191, + "total_ms": 1546.1869998835027 + }, + { + "extract_ms": 1672.652999870479, + "serialize_ms": 68.25180002488196, + "total_ms": 1740.904799895361 + }, + { + "extract_ms": 1523.951900191605, + "serialize_ms": 71.91520021297038, + "total_ms": 1595.8671004045755 + } + ], + "repeated_median": { + "extract_ms": 1523.951900191605, + "serialize_ms": 71.91520021297038, + "total_ms": 1595.8671004045755 + }, + "peak_rss_bytes": 198279168, + "output_bytes": 788238, + "profile": { + "all_features": false, + "extract_ms": 1625.7439001929015, + "serialize_ms": 70.50179992802441, + "total_ms": 1696.245700120926, + "stages": { + "cell_extraction": { + "ms": 322.4330998491496, + "calls": 1 + }, + "print_area_extraction": { + "ms": 486.2160000484437, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 816.1366998683661, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.6146002560853958, + "calls": 2 + }, + "model_construction": { + "ms": 0.11699995957314968, + "calls": 1 + }, + "workbook_parsing": { + "ms": 1287.4598000198603, + "calls": 4 + } + }, + "openpyxl_workbook_opens": 4, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "62de4a14fcba921db15a1715a2215d1bd88310ff520daeef4a2b0d91c555362c" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:55:02.253109+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "large.xlsx", + "sha256": "72ff66fa3abde812e9ae3261530b1dd5b88c4dfd5173715b383d896b990f1b97", + "size_bytes": 185797 + }, + "mode": "standard", + "startup": { + "python_process_ms": 40.24049988947809, + "package_import_process_ms": 72.41270015947521, + "cli_help_process_ms": 80.3173000458628, + "cold_extraction_process_ms": 5847.524499986321, + "cold_extraction_in_process_ms": 5164.6980999503285, + "measurement_worker_process_ms": 35051.4255000744 + }, + "import_ms": 5.982200149446726, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 7573.030299972743, + "serialize_ms": 88.16149993799627, + "total_ms": 7661.191799910739 + }, + "repeated": [ + { + "extract_ms": 11792.07029985264, + "serialize_ms": 86.65130008012056, + "total_ms": 11878.72159993276 + }, + { + "extract_ms": 5523.50450004451, + "serialize_ms": 82.0102000143379, + "total_ms": 5605.514700058848 + }, + { + "extract_ms": 4200.71589993313, + "serialize_ms": 84.89860012196004, + "total_ms": 4285.61450005509 + } + ], + "repeated_median": { + "extract_ms": 5523.50450004451, + "serialize_ms": 84.89860012196004, + "total_ms": 5605.514700058848 + }, + "peak_rss_bytes": 196755456, + "output_bytes": 788238, + "profile": { + "all_features": false, + "extract_ms": 4426.707600010559, + "serialize_ms": 85.37060022354126, + "total_ms": 4512.0782002341, + "stages": { + "cell_extraction": { + "ms": 461.8561000097543, + "calls": 1 + }, + "print_area_extraction": { + "ms": 484.8545000422746, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 496.0709000006318, + "calls": 1 + }, + "table_detection": { + "ms": 1109.7055999562144, + "calls": 1 + }, + "rich_extraction": { + "ms": 23.934299824759364, + "calls": 2 + }, + "model_construction": { + "ms": 0.24030008353292942, + "calls": 1 + }, + "workbook_parsing": { + "ms": 2064.6241998765618, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "62de4a14fcba921db15a1715a2215d1bd88310ff520daeef4a2b0d91c555362c" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:55:47.232610+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "many-sheet.xlsx", + "sha256": "608cf4728344614879790a083cf641109dfbed26dc9eafc60002fd75ae667ce0", + "size_bytes": 42154 + }, + "mode": "light", + "startup": { + "python_process_ms": 36.211400059983134, + "package_import_process_ms": 75.64759999513626, + "cli_help_process_ms": 80.19899995997548, + "cold_extraction_process_ms": 5120.132099837065, + "cold_extraction_in_process_ms": 4313.559600152075, + "measurement_worker_process_ms": 21815.90999988839 + }, + "import_ms": 4.471000051125884, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 5396.298799896613, + "serialize_ms": 14.476400101557374, + "total_ms": 5410.77519999817 + }, + "repeated": [ + { + "extract_ms": 4090.1345999445766, + "serialize_ms": 11.628499953076243, + "total_ms": 4101.763099897653 + }, + { + "extract_ms": 4440.678099868819, + "serialize_ms": 10.714000090956688, + "total_ms": 4451.392099959776 + }, + { + "extract_ms": 3520.2567998785526, + "serialize_ms": 10.88219997473061, + "total_ms": 3531.138999853283 + } + ], + "repeated_median": { + "extract_ms": 4090.1345999445766, + "serialize_ms": 10.88219997473061, + "total_ms": 4101.763099897653 + }, + "peak_rss_bytes": 139702272, + "output_bytes": 86252, + "profile": { + "all_features": false, + "extract_ms": 3547.60879999958, + "serialize_ms": 10.927399853244424, + "total_ms": 3558.536199852824, + "stages": { + "cell_extraction": { + "ms": 159.77940009906888, + "calls": 1 + }, + "print_area_extraction": { + "ms": 48.247900092974305, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 3337.9490999504924, + "calls": 24 + }, + "rich_extraction": { + "ms": 0.9268999565392733, + "calls": 2 + }, + "model_construction": { + "ms": 0.13679987750947475, + "calls": 1 + }, + "workbook_parsing": { + "ms": 3454.2037998326123, + "calls": 50 + } + }, + "openpyxl_workbook_opens": 50, + "archive_opens": 52, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "e5b99cbf8adf3e91b6e06b71cf8143a2354a4b1775f46792990bb40a0b21b41c" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:56:30.722494+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "many-sheet.xlsx", + "sha256": "608cf4728344614879790a083cf641109dfbed26dc9eafc60002fd75ae667ce0", + "size_bytes": 42154 + }, + "mode": "standard", + "startup": { + "python_process_ms": 45.08079984225333, + "package_import_process_ms": 94.29230005480349, + "cli_help_process_ms": 121.81190005503595, + "cold_extraction_process_ms": 8134.281100006774, + "cold_extraction_in_process_ms": 7447.201899951324, + "measurement_worker_process_ms": 31780.674600042403 + }, + "import_ms": 6.931300042197108, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 7625.4618999082595, + "serialize_ms": 21.856300067156553, + "total_ms": 7647.318199975416 + }, + "repeated": [ + { + "extract_ms": 6120.882400078699, + "serialize_ms": 12.104999972507358, + "total_ms": 6132.987400051206 + }, + { + "extract_ms": 5308.429099852219, + "serialize_ms": 14.977400191128254, + "total_ms": 5323.4065000433475 + }, + { + "extract_ms": 6226.717400131747, + "serialize_ms": 10.352599900215864, + "total_ms": 6237.070000031963 + } + ], + "repeated_median": { + "extract_ms": 6120.882400078699, + "serialize_ms": 12.104999972507358, + "total_ms": 6132.987400051206 + }, + "peak_rss_bytes": 144519168, + "output_bytes": 86252, + "profile": { + "all_features": false, + "extract_ms": 5625.45570009388, + "serialize_ms": 12.012500083073974, + "total_ms": 5637.468200176954, + "stages": { + "cell_extraction": { + "ms": 139.11709981039166, + "calls": 1 + }, + "print_area_extraction": { + "ms": 50.495600095018744, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 51.760099828243256, + "calls": 1 + }, + "table_detection": { + "ms": 3583.071099827066, + "calls": 24 + }, + "rich_extraction": { + "ms": 201.14519982598722, + "calls": 2 + }, + "model_construction": { + "ms": 0.16130018047988415, + "calls": 1 + }, + "workbook_parsing": { + "ms": 3679.780500009656, + "calls": 51 + } + }, + "openpyxl_workbook_opens": 51, + "archive_opens": 52, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "e5b99cbf8adf3e91b6e06b71cf8143a2354a4b1775f46792990bb40a0b21b41c" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:56:50.728780+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "small.xlsx", + "sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "size_bytes": 7221 + }, + "mode": "light", + "startup": { + "python_process_ms": 32.89669984951615, + "package_import_process_ms": 66.47550011985004, + "cli_help_process_ms": 78.52749992161989, + "cold_extraction_process_ms": 1552.6005998253822, + "cold_extraction_in_process_ms": 929.3563000392169, + "measurement_worker_process_ms": 1650.5291999783367 + }, + "import_ms": 6.975499913096428, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 935.0596000440419, + "serialize_ms": 2.5673999916762114, + "total_ms": 937.6270000357181 + }, + "repeated": [ + { + "extract_ms": 22.756400052458048, + "serialize_ms": 0.3879999276250601, + "total_ms": 23.144399980083108 + }, + { + "extract_ms": 24.380400078371167, + "serialize_ms": 0.43059978634119034, + "total_ms": 24.810999864712358 + }, + { + "extract_ms": 20.964100025594234, + "serialize_ms": 0.31579984351992607, + "total_ms": 21.27989986911416 + } + ], + "repeated_median": { + "extract_ms": 22.756400052458048, + "serialize_ms": 0.3879999276250601, + "total_ms": 23.144399980083108 + }, + "peak_rss_bytes": 122261504, + "output_bytes": 2305, + "profile": { + "all_features": false, + "extract_ms": 20.36729990504682, + "serialize_ms": 0.3921000752598047, + "total_ms": 20.759399980306625, + "stages": { + "cell_extraction": { + "ms": 5.2143000066280365, + "calls": 1 + }, + "print_area_extraction": { + "ms": 3.8120001554489136, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 9.715499822050333, + "calls": 1 + }, + "rich_extraction": { + "ms": 1.4198999851942062, + "calls": 2 + }, + "model_construction": { + "ms": 0.034700147807598114, + "calls": 1 + }, + "workbook_parsing": { + "ms": 15.129300067201257, + "calls": 4 + } + }, + "openpyxl_workbook_opens": 4, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "ce4d694ba4a42d05d55e33d6811e145f9e5a2d68512bd50b9c6140d064b5d1b4" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:57:08.070888+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "small.xlsx", + "sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "size_bytes": 7221 + }, + "mode": "standard", + "startup": { + "python_process_ms": 32.17629995197058, + "package_import_process_ms": 75.28079999610782, + "cli_help_process_ms": 78.13869998790324, + "cold_extraction_process_ms": 3358.8670999743044, + "cold_extraction_in_process_ms": 2589.513000100851, + "measurement_worker_process_ms": 11407.942000078037 + }, + "import_ms": 4.945100052282214, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 2676.7614998389035, + "serialize_ms": 3.913300111889839, + "total_ms": 2680.6747999507934 + }, + "repeated": [ + { + "extract_ms": 1894.6800001431257, + "serialize_ms": 0.6434998940676451, + "total_ms": 1895.3235000371933 + }, + { + "extract_ms": 1960.5547001119703, + "serialize_ms": 0.6251002196222544, + "total_ms": 1961.1798003315926 + }, + { + "extract_ms": 1962.8351998981088, + "serialize_ms": 0.48169982619583607, + "total_ms": 1963.3168997243047 + } + ], + "repeated_median": { + "extract_ms": 1960.5547001119703, + "serialize_ms": 0.6251002196222544, + "total_ms": 1961.1798003315926 + }, + "peak_rss_bytes": 128028672, + "output_bytes": 2422, + "profile": { + "all_features": false, + "extract_ms": 2122.846700018272, + "serialize_ms": 0.48459996469318867, + "total_ms": 2123.331299982965, + "stages": { + "cell_extraction": { + "ms": 9.55060007981956, + "calls": 1 + }, + "print_area_extraction": { + "ms": 4.9270999152213335, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 5.620399955660105, + "calls": 1 + }, + "table_detection": { + "ms": 14.115599915385246, + "calls": 1 + }, + "rich_extraction": { + "ms": 41.26910003833473, + "calls": 2 + }, + "model_construction": { + "ms": 0.10499986819922924, + "calls": 1 + }, + "workbook_parsing": { + "ms": 25.881699984893203, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "bd64b54a5359ef2c99a9d10c10e431ad43d3658b3aa48a83f29dff82e8a0fbe6" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:57:27.105073+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "sparse.xlsx", + "sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "size_bytes": 5417 + }, + "mode": "light", + "startup": { + "python_process_ms": 37.408600095659494, + "package_import_process_ms": 66.30219984799623, + "cli_help_process_ms": 86.33520011790097, + "cold_extraction_process_ms": 2173.2097999192774, + "cold_extraction_in_process_ms": 1246.5524999424815, + "measurement_worker_process_ms": 2528.8367001339793 + }, + "import_ms": 6.376500008627772, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 1277.98720006831, + "serialize_ms": 2.8593000024557114, + "total_ms": 1280.8465000707656 + }, + "repeated": [ + { + "extract_ms": 143.03559996187687, + "serialize_ms": 0.2934001386165619, + "total_ms": 143.32900010049343 + }, + { + "extract_ms": 65.7774000428617, + "serialize_ms": 0.36399997770786285, + "total_ms": 66.14140002056956 + }, + { + "extract_ms": 69.94149996899068, + "serialize_ms": 0.2625999040901661, + "total_ms": 70.20409987308085 + } + ], + "repeated_median": { + "extract_ms": 69.94149996899068, + "serialize_ms": 0.2934001386165619, + "total_ms": 70.20409987308085 + }, + "peak_rss_bytes": 132317184, + "output_bytes": 1846, + "profile": { + "all_features": false, + "extract_ms": 68.1823999620974, + "serialize_ms": 0.28280005790293217, + "total_ms": 68.46520002000034, + "stages": { + "cell_extraction": { + "ms": 21.91480016335845, + "calls": 1 + }, + "print_area_extraction": { + "ms": 3.9858000818639994, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 41.51080013252795, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.426099868491292, + "calls": 2 + }, + "model_construction": { + "ms": 0.04689977504312992, + "calls": 1 + }, + "workbook_parsing": { + "ms": 12.18960015103221, + "calls": 4 + } + }, + "openpyxl_workbook_opens": 4, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "d581419f5c4c92d17f0680515c85cc175325852ebc16c495258619926e1a2d43" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:57:45.706915+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "sparse.xlsx", + "sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "size_bytes": 5417 + }, + "mode": "standard", + "startup": { + "python_process_ms": 44.21349987387657, + "package_import_process_ms": 106.35460005141795, + "cli_help_process_ms": 121.20319996029139, + "cold_extraction_process_ms": 3715.8335000276566, + "cold_extraction_in_process_ms": 2934.768799925223, + "measurement_worker_process_ms": 10600.324800005183 + }, + "import_ms": 9.006500011309981, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3329.2708001099527, + "serialize_ms": 3.7263999693095684, + "total_ms": 3332.9972000792623 + }, + "repeated": [ + { + "extract_ms": 1659.8763999063522, + "serialize_ms": 0.3969999961555004, + "total_ms": 1660.2733999025077 + }, + { + "extract_ms": 1722.9073999915272, + "serialize_ms": 0.29790005646646023, + "total_ms": 1723.2053000479937 + }, + { + "extract_ms": 1570.6287000793964, + "serialize_ms": 0.35949982702732086, + "total_ms": 1570.9881999064237 + } + ], + "repeated_median": { + "extract_ms": 1659.8763999063522, + "serialize_ms": 0.35949982702732086, + "total_ms": 1660.2733999025077 + }, + "peak_rss_bytes": 137138176, + "output_bytes": 1846, + "profile": { + "all_features": false, + "extract_ms": 1537.1107999235392, + "serialize_ms": 0.2959000412374735, + "total_ms": 1537.4066999647766, + "stages": { + "cell_extraction": { + "ms": 21.528800018131733, + "calls": 1 + }, + "print_area_extraction": { + "ms": 3.661900060251355, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 3.384700044989586, + "calls": 1 + }, + "table_detection": { + "ms": 98.8064999692142, + "calls": 1 + }, + "rich_extraction": { + "ms": 18.25250033289194, + "calls": 2 + }, + "model_construction": { + "ms": 0.036300159990787506, + "calls": 1 + }, + "workbook_parsing": { + "ms": 15.214899787679315, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "d581419f5c4c92d17f0680515c85cc175325852ebc16c495258619926e1a2d43" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:58:08.131106+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "style-heavy.xlsx", + "sha256": "05a77cc146594f2db73cb3aa764ad2625fa798809fce82fbe42900b5cc49529e", + "size_bytes": 38626 + }, + "mode": "light", + "startup": { + "python_process_ms": 47.42970014922321, + "package_import_process_ms": 94.42580002360046, + "cli_help_process_ms": 121.5307000093162, + "cold_extraction_process_ms": 1905.0881001167, + "cold_extraction_in_process_ms": 1329.2163000442088, + "measurement_worker_process_ms": 4628.251200076193 + }, + "import_ms": 8.390799863263965, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 1720.7557999063283, + "serialize_ms": 9.816600009799004, + "total_ms": 1730.5723999161273 + }, + "repeated": [ + { + "extract_ms": 525.9386000689119, + "serialize_ms": 9.477300103753805, + "total_ms": 535.4159001726657 + }, + { + "extract_ms": 346.7371999286115, + "serialize_ms": 8.248499827459455, + "total_ms": 354.98569975607097 + }, + { + "extract_ms": 582.3312001302838, + "serialize_ms": 10.232699802145362, + "total_ms": 592.5638999324292 + } + ], + "repeated_median": { + "extract_ms": 525.9386000689119, + "serialize_ms": 9.477300103753805, + "total_ms": 535.4159001726657 + }, + "peak_rss_bytes": 134275072, + "output_bytes": 73844, + "profile": { + "all_features": false, + "extract_ms": 395.83919988945127, + "serialize_ms": 9.76069993339479, + "total_ms": 405.59989982284606, + "stages": { + "cell_extraction": { + "ms": 81.38530002906919, + "calls": 1 + }, + "print_area_extraction": { + "ms": 145.51490009762347, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 167.8645999636501, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.5999000277370214, + "calls": 2 + }, + "model_construction": { + "ms": 0.07269997149705887, + "calls": 1 + }, + "workbook_parsing": { + "ms": 320.56690007448196, + "calls": 4 + } + }, + "openpyxl_workbook_opens": 4, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "d15601bca18dbc97a66883ec4f81fc3ea8ec113849a746abdf97cb7db4a943f3" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:58:27.903567+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "style-heavy.xlsx", + "sha256": "05a77cc146594f2db73cb3aa764ad2625fa798809fce82fbe42900b5cc49529e", + "size_bytes": 38626 + }, + "mode": "standard", + "startup": { + "python_process_ms": 52.29509994387627, + "package_import_process_ms": 95.4148001037538, + "cli_help_process_ms": 102.48779994435608, + "cold_extraction_process_ms": 4321.904600132257, + "cold_extraction_in_process_ms": 3496.619699988514, + "measurement_worker_process_ms": 11925.701000029221 + }, + "import_ms": 5.25819999165833, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3336.2479999195784, + "serialize_ms": 11.088000144809484, + "total_ms": 3347.336000064388 + }, + "repeated": [ + { + "extract_ms": 2136.3647000398487, + "serialize_ms": 8.038700092583895, + "total_ms": 2144.4034001324326 + }, + { + "extract_ms": 1860.9563999343663, + "serialize_ms": 7.473599864169955, + "total_ms": 1868.4299997985363 + }, + { + "extract_ms": 1899.9750001821667, + "serialize_ms": 7.830000016838312, + "total_ms": 1907.805000199005 + } + ], + "repeated_median": { + "extract_ms": 1899.9750001821667, + "serialize_ms": 7.830000016838312, + "total_ms": 1907.805000199005 + }, + "peak_rss_bytes": 141889536, + "output_bytes": 73844, + "profile": { + "all_features": false, + "extract_ms": 1957.2821001056582, + "serialize_ms": 7.865100167691708, + "total_ms": 1965.1472002733499, + "stages": { + "cell_extraction": { + "ms": 101.92919988185167, + "calls": 1 + }, + "print_area_extraction": { + "ms": 56.81780003942549, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 56.40000011771917, + "calls": 1 + }, + "table_detection": { + "ms": 241.40439997427166, + "calls": 1 + }, + "rich_extraction": { + "ms": 31.47789998911321, + "calls": 2 + }, + "model_construction": { + "ms": 0.1113000325858593, + "calls": 1 + }, + "workbook_parsing": { + "ms": 385.0426999852061, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "d15601bca18dbc97a66883ec4f81fc3ea8ec113849a746abdf97cb7db4a943f3" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:58:48.553054+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "table-heavy.xlsx", + "sha256": "a319c15ab9b4987527beaa627ca336a679e9091f34427c08710199cfda776e5f", + "size_bytes": 38745 + }, + "mode": "light", + "startup": { + "python_process_ms": 44.01920014061034, + "package_import_process_ms": 79.09339992329478, + "cli_help_process_ms": 93.77690008841455, + "cold_extraction_process_ms": 2301.016799872741, + "cold_extraction_in_process_ms": 1456.6870999988168, + "measurement_worker_process_ms": 3069.688099902123 + }, + "import_ms": 8.62679979763925, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 1261.7007000371814, + "serialize_ms": 12.94899987988174, + "total_ms": 1274.6496999170631 + }, + "repeated": [ + { + "extract_ms": 239.78900001384318, + "serialize_ms": 10.084599955007434, + "total_ms": 249.8735999688506 + }, + { + "extract_ms": 227.2520000115037, + "serialize_ms": 10.300899855792522, + "total_ms": 237.55289986729622 + }, + { + "extract_ms": 225.72539979591966, + "serialize_ms": 16.360699897632003, + "total_ms": 242.08609969355166 + } + ], + "repeated_median": { + "extract_ms": 227.2520000115037, + "serialize_ms": 10.300899855792522, + "total_ms": 242.08609969355166 + }, + "peak_rss_bytes": 136241152, + "output_bytes": 92326, + "profile": { + "all_features": false, + "extract_ms": 225.46129999682307, + "serialize_ms": 10.010699974372983, + "total_ms": 235.47199997119606, + "stages": { + "cell_extraction": { + "ms": 92.6260999403894, + "calls": 1 + }, + "print_area_extraction": { + "ms": 40.531399892643094, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 91.28820011392236, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.7087001577019691, + "calls": 2 + }, + "model_construction": { + "ms": 0.04529999569058418, + "calls": 1 + }, + "workbook_parsing": { + "ms": 130.1253002602607, + "calls": 4 + } + }, + "openpyxl_workbook_opens": 4, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "46649d4d0baeb567b1bf589ae5cd294688787938cb695f1c8802efd9d50282d7" + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T05:59:09.230086+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "table-heavy.xlsx", + "sha256": "a319c15ab9b4987527beaa627ca336a679e9091f34427c08710199cfda776e5f", + "size_bytes": 38745 + }, + "mode": "standard", + "startup": { + "python_process_ms": 43.586800107732415, + "package_import_process_ms": 84.22830002382398, + "cli_help_process_ms": 87.37919991835952, + "cold_extraction_process_ms": 3800.39309989661, + "cold_extraction_in_process_ms": 3166.677499888465, + "measurement_worker_process_ms": 13413.959299912676 + }, + "import_ms": 6.591999903321266, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3334.5868000760674, + "serialize_ms": 16.72989991493523, + "total_ms": 3351.3166999910027 + }, + "repeated": [ + { + "extract_ms": 2476.7772001214325, + "serialize_ms": 13.504800153896213, + "total_ms": 2490.2820002753288 + }, + { + "extract_ms": 2090.295400004834, + "serialize_ms": 11.455799918621778, + "total_ms": 2101.7511999234557 + }, + { + "extract_ms": 2309.1228001285344, + "serialize_ms": 16.201899852603674, + "total_ms": 2325.324699981138 + } + ], + "repeated_median": { + "extract_ms": 2309.1228001285344, + "serialize_ms": 13.504800153896213, + "total_ms": 2325.324699981138 + }, + "peak_rss_bytes": 142950400, + "output_bytes": 92326, + "profile": { + "all_features": false, + "extract_ms": 2342.3374998383224, + "serialize_ms": 13.524799840524793, + "total_ms": 2355.862299678847, + "stages": { + "cell_extraction": { + "ms": 202.04430003650486, + "calls": 1 + }, + "print_area_extraction": { + "ms": 53.06370020844042, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 67.41419993340969, + "calls": 1 + }, + "table_detection": { + "ms": 166.64259997196496, + "calls": 1 + }, + "rich_extraction": { + "ms": 26.57989994622767, + "calls": 2 + }, + "model_construction": { + "ms": 0.08080014958977699, + "calls": 1 + }, + "workbook_parsing": { + "ms": 280.4798998404294, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "46649d4d0baeb567b1bf589ae5cd294688787938cb695f1c8802efd9d50282d7" + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:05:03.096439+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "large.xlsx", + "sha256": "72ff66fa3abde812e9ae3261530b1dd5b88c4dfd5173715b383d896b990f1b97", + "size_bytes": 185797 + }, + "mode": "light", + "startup": { + "python_process_ms": 32.57409995421767, + "package_import_process_ms": 68.96070018410683, + "cli_help_process_ms": 79.43640020675957, + "cold_extraction_process_ms": 1611.8250000290573, + "cold_extraction_in_process_ms": 1380.7361000217497, + "measurement_worker_process_ms": 3974.600999848917 + }, + "import_ms": 4.56439983099699, + "modules_after_import": [], + "modules_after_extraction": [ + "numpy", + "scipy", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 1024.3405001237988, + "serialize_ms": 135.7191998977214, + "total_ms": 1160.0597000215203 + }, + "repeated": [ + { + "extract_ms": 455.76979988254607, + "serialize_ms": 78.569799894467, + "total_ms": 534.3395997770131 + }, + { + "extract_ms": 521.0869000293314, + "serialize_ms": 75.20200009457767, + "total_ms": 596.2889001239091 + }, + { + "extract_ms": 596.2008000351489, + "serialize_ms": 74.09010012634099, + "total_ms": 670.2909001614898 + } + ], + "repeated_median": { + "extract_ms": 521.0869000293314, + "serialize_ms": 75.20200009457767, + "total_ms": 596.2889001239091 + }, + "peak_rss_bytes": 114626560, + "output_bytes": 788238, + "profile": { + "all_features": false, + "extract_ms": 594.4193000905216, + "serialize_ms": 76.85050019063056, + "total_ms": 671.2698002811521, + "stages": { + "cell_extraction": { + "ms": 570.201100083068, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.1683998852968216, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 22.381999995559454, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.8992000948637724, + "calls": 2 + }, + "model_construction": { + "ms": 0.2850999590009451, + "calls": 1 + }, + "workbook_parsing": { + "ms": 443.00649990327656, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 2, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "62de4a14fcba921db15a1715a2215d1bd88310ff520daeef4a2b0d91c555362c", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:05:27.435107+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "large.xlsx", + "sha256": "72ff66fa3abde812e9ae3261530b1dd5b88c4dfd5173715b383d896b990f1b97", + "size_bytes": 185797 + }, + "mode": "standard", + "startup": { + "python_process_ms": 37.71309996955097, + "package_import_process_ms": 69.32749995030463, + "cli_help_process_ms": 90.93230008147657, + "cold_extraction_process_ms": 4126.265099970624, + "cold_extraction_in_process_ms": 3343.564199982211, + "measurement_worker_process_ms": 13753.190899966285 + }, + "import_ms": 7.54119991324842, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3055.991099914536, + "serialize_ms": 80.36340004764497, + "total_ms": 3136.354499962181 + }, + "repeated": [ + { + "extract_ms": 2309.8116000182927, + "serialize_ms": 185.1387000642717, + "total_ms": 2494.9503000825644 + }, + { + "extract_ms": 2278.1177998986095, + "serialize_ms": 75.2208000048995, + "total_ms": 2353.338599903509 + }, + { + "extract_ms": 2567.340799840167, + "serialize_ms": 90.05020000040531, + "total_ms": 2657.3909998405725 + } + ], + "repeated_median": { + "extract_ms": 2309.8116000182927, + "serialize_ms": 90.05020000040531, + "total_ms": 2494.9503000825644 + }, + "peak_rss_bytes": 173985792, + "output_bytes": 788238, + "profile": { + "all_features": false, + "extract_ms": 2221.30350000225, + "serialize_ms": 77.86530000157654, + "total_ms": 2299.1688000038266, + "stages": { + "cell_extraction": { + "ms": 530.9213998261839, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.09329989552497864, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.007700175046920776, + "calls": 1 + }, + "table_detection": { + "ms": 28.675599955022335, + "calls": 1 + }, + "rich_extraction": { + "ms": 19.25299991853535, + "calls": 2 + }, + "model_construction": { + "ms": 0.5136001855134964, + "calls": 1 + }, + "workbook_parsing": { + "ms": 398.9458999130875, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "62de4a14fcba921db15a1715a2215d1bd88310ff520daeef4a2b0d91c555362c", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:05:41.712139+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "many-sheet.xlsx", + "sha256": "608cf4728344614879790a083cf641109dfbed26dc9eafc60002fd75ae667ce0", + "size_bytes": 42154 + }, + "mode": "light", + "startup": { + "python_process_ms": 40.969999972730875, + "package_import_process_ms": 74.6715001296252, + "cli_help_process_ms": 82.15899998322129, + "cold_extraction_process_ms": 1029.6004998963326, + "cold_extraction_in_process_ms": 859.7548000980169, + "measurement_worker_process_ms": 1479.1457999963313 + }, + "import_ms": 4.968099994584918, + "modules_after_import": [], + "modules_after_extraction": [ + "numpy", + "scipy", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 652.6506999507546, + "serialize_ms": 12.957200175151229, + "total_ms": 665.6079001259059 + }, + "repeated": [ + { + "extract_ms": 135.67009987309575, + "serialize_ms": 15.616500051692128, + "total_ms": 151.28659992478788 + }, + { + "extract_ms": 106.3549998216331, + "serialize_ms": 13.133199885487556, + "total_ms": 119.48819970712066 + }, + { + "extract_ms": 95.95569991506636, + "serialize_ms": 10.726399952545762, + "total_ms": 106.68209986761212 + } + ], + "repeated_median": { + "extract_ms": 106.3549998216331, + "serialize_ms": 13.133199885487556, + "total_ms": 119.48819970712066 + }, + "peak_rss_bytes": 79237120, + "output_bytes": 86252, + "profile": { + "all_features": false, + "extract_ms": 92.67619997262955, + "serialize_ms": 10.594800114631653, + "total_ms": 103.2710000872612, + "stages": { + "cell_extraction": { + "ms": 68.87260009534657, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.3470000810921192, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 21.502199349924922, + "calls": 24 + }, + "rich_extraction": { + "ms": 1.3289002235978842, + "calls": 2 + }, + "model_construction": { + "ms": 0.1892000436782837, + "calls": 1 + }, + "workbook_parsing": { + "ms": 56.474899873137474, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 2, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "e5b99cbf8adf3e91b6e06b71cf8143a2354a4b1775f46792990bb40a0b21b41c", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:06:08.810506+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "many-sheet.xlsx", + "sha256": "608cf4728344614879790a083cf641109dfbed26dc9eafc60002fd75ae667ce0", + "size_bytes": 42154 + }, + "mode": "standard", + "startup": { + "python_process_ms": 39.303400088101625, + "package_import_process_ms": 77.82030012458563, + "cli_help_process_ms": 88.31809996627271, + "cold_extraction_process_ms": 4638.2057999726385, + "cold_extraction_in_process_ms": 3759.990300051868, + "measurement_worker_process_ms": 14836.865599965677 + }, + "import_ms": 5.374500062316656, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3983.2188999280334, + "serialize_ms": 13.129200087860227, + "total_ms": 3996.3481000158936 + }, + "repeated": [ + { + "extract_ms": 2473.9355000201613, + "serialize_ms": 11.36410003527999, + "total_ms": 2485.2996000554413 + }, + { + "extract_ms": 2168.858499964699, + "serialize_ms": 12.06470001488924, + "total_ms": 2180.9231999795884 + }, + { + "extract_ms": 2554.4497000519186, + "serialize_ms": 14.114100020378828, + "total_ms": 2568.5638000722975 + } + ], + "repeated_median": { + "extract_ms": 2473.9355000201613, + "serialize_ms": 12.06470001488924, + "total_ms": 2485.2996000554413 + }, + "peak_rss_bytes": 135618560, + "output_bytes": 86252, + "profile": { + "all_features": false, + "extract_ms": 2587.070500012487, + "serialize_ms": 13.692100066691637, + "total_ms": 2600.762600079179, + "stages": { + "cell_extraction": { + "ms": 62.19449988566339, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.25669997557997704, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.018700025975704193, + "calls": 1 + }, + "table_detection": { + "ms": 56.246999418362975, + "calls": 24 + }, + "rich_extraction": { + "ms": 198.67249974049628, + "calls": 2 + }, + "model_construction": { + "ms": 0.14320015907287598, + "calls": 1 + }, + "workbook_parsing": { + "ms": 54.74360007792711, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "e5b99cbf8adf3e91b6e06b71cf8143a2354a4b1775f46792990bb40a0b21b41c", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:06:23.649098+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "small.xlsx", + "sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "size_bytes": 7221 + }, + "mode": "light", + "startup": { + "python_process_ms": 32.94200007803738, + "package_import_process_ms": 72.64019991271198, + "cli_help_process_ms": 87.43039984256029, + "cold_extraction_process_ms": 850.5188999697566, + "cold_extraction_in_process_ms": 629.9773000646383, + "measurement_worker_process_ms": 916.9727999251336 + }, + "import_ms": 4.855500068515539, + "modules_after_import": [], + "modules_after_extraction": [ + "numpy", + "scipy", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 643.6773000750691, + "serialize_ms": 2.526399912312627, + "total_ms": 646.2036999873817 + }, + "repeated": [ + { + "extract_ms": 7.4155000038445, + "serialize_ms": 0.34760008566081524, + "total_ms": 7.763100089505315 + }, + { + "extract_ms": 6.759499898180366, + "serialize_ms": 0.36060018464922905, + "total_ms": 7.120100082829595 + }, + { + "extract_ms": 7.060500094667077, + "serialize_ms": 0.33740000799298286, + "total_ms": 7.39790010266006 + } + ], + "repeated_median": { + "extract_ms": 7.060500094667077, + "serialize_ms": 0.34760008566081524, + "total_ms": 7.39790010266006 + }, + "peak_rss_bytes": 67432448, + "output_bytes": 2305, + "profile": { + "all_features": false, + "extract_ms": 6.51129987090826, + "serialize_ms": 0.32020010985434055, + "total_ms": 6.831499980762601, + "stages": { + "cell_extraction": { + "ms": 4.301500041037798, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.03660004585981369, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 0.5073999054729939, + "calls": 1 + }, + "rich_extraction": { + "ms": 1.487199915573001, + "calls": 2 + }, + "model_construction": { + "ms": 0.015799887478351593, + "calls": 1 + }, + "workbook_parsing": { + "ms": 4.046000074595213, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 2, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "ce4d694ba4a42d05d55e33d6811e145f9e5a2d68512bd50b9c6140d064b5d1b4", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:06:44.170413+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "small.xlsx", + "sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "size_bytes": 7221 + }, + "mode": "standard", + "startup": { + "python_process_ms": 39.21610000543296, + "package_import_process_ms": 73.3419000171125, + "cli_help_process_ms": 100.12180008925498, + "cold_extraction_process_ms": 3783.4013998508453, + "cold_extraction_in_process_ms": 2843.838300090283, + "measurement_worker_process_ms": 13781.828599981964 + }, + "import_ms": 7.026100065559149, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3448.7602000590414, + "serialize_ms": 3.4643998369574547, + "total_ms": 3452.224599895999 + }, + "repeated": [ + { + "extract_ms": 2783.2371999975294, + "serialize_ms": 0.8853001054376364, + "total_ms": 2784.122500102967 + }, + { + "extract_ms": 2499.9865000136197, + "serialize_ms": 0.5493001081049442, + "total_ms": 2500.5358001217246 + }, + { + "extract_ms": 2008.5084000602365, + "serialize_ms": 0.593000091612339, + "total_ms": 2009.1014001518488 + } + ], + "repeated_median": { + "extract_ms": 2499.9865000136197, + "serialize_ms": 0.593000091612339, + "total_ms": 2500.5358001217246 + }, + "peak_rss_bytes": 126918656, + "output_bytes": 2422, + "profile": { + "all_features": false, + "extract_ms": 2110.516499960795, + "serialize_ms": 0.5838999059051275, + "total_ms": 2111.1003998667, + "stages": { + "cell_extraction": { + "ms": 5.449800053611398, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.08149980567395687, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.019200146198272705, + "calls": 1 + }, + "table_detection": { + "ms": 2.6702999603003263, + "calls": 1 + }, + "rich_extraction": { + "ms": 37.44570002891123, + "calls": 2 + }, + "model_construction": { + "ms": 0.11679995805025101, + "calls": 1 + }, + "workbook_parsing": { + "ms": 5.1575000397861, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "bd64b54a5359ef2c99a9d10c10e431ad43d3658b3aa48a83f29dff82e8a0fbe6", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:06:56.467552+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "sparse.xlsx", + "sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "size_bytes": 5417 + }, + "mode": "light", + "startup": { + "python_process_ms": 50.43700011447072, + "package_import_process_ms": 93.79630000330508, + "cli_help_process_ms": 115.08170003071427, + "cold_extraction_process_ms": 1337.8595998510718, + "cold_extraction_in_process_ms": 1096.1811998859048, + "measurement_worker_process_ms": 1199.0424001123756 + }, + "import_ms": 5.7200000155717134, + "modules_after_import": [], + "modules_after_extraction": [ + "numpy", + "scipy", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 681.2096000649035, + "serialize_ms": 2.212400082498789, + "total_ms": 683.4220001474023 + }, + "repeated": [ + { + "extract_ms": 41.17449978366494, + "serialize_ms": 0.2576999831944704, + "total_ms": 41.43219976685941 + }, + { + "extract_ms": 78.53420008905232, + "serialize_ms": 0.3029999788850546, + "total_ms": 78.83720006793737 + }, + { + "extract_ms": 39.325099904090166, + "serialize_ms": 0.3698000218719244, + "total_ms": 39.69489992596209 + } + ], + "repeated_median": { + "extract_ms": 41.17449978366494, + "serialize_ms": 0.3029999788850546, + "total_ms": 41.43219976685941 + }, + "peak_rss_bytes": 74502144, + "output_bytes": 1846, + "profile": { + "all_features": false, + "extract_ms": 40.79320002347231, + "serialize_ms": 0.2617000136524439, + "total_ms": 41.05490003712475, + "stages": { + "cell_extraction": { + "ms": 3.8478998467326164, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.039300182834267616, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 36.13519994542003, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.44599990360438824, + "calls": 2 + }, + "model_construction": { + "ms": 0.033800024539232254, + "calls": 1 + }, + "workbook_parsing": { + "ms": 3.636399982497096, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 2, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "d581419f5c4c92d17f0680515c85cc175325852ebc16c495258619926e1a2d43", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:07:22.935754+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "sparse.xlsx", + "sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "size_bytes": 5417 + }, + "mode": "standard", + "startup": { + "python_process_ms": 47.927800100296736, + "package_import_process_ms": 81.55749994330108, + "cli_help_process_ms": 121.75099994055927, + "cold_extraction_process_ms": 4638.722299830988, + "cold_extraction_in_process_ms": 3686.697500059381, + "measurement_worker_process_ms": 17544.33450009674 + }, + "import_ms": 9.327900130301714, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3673.049199860543, + "serialize_ms": 3.684199880808592, + "total_ms": 3676.7333997413516 + }, + "repeated": [ + { + "extract_ms": 2333.723100135103, + "serialize_ms": 0.44109998270869255, + "total_ms": 2334.1642001178116 + }, + { + "extract_ms": 2381.4179999753833, + "serialize_ms": 0.5778998602181673, + "total_ms": 2381.9958998356014 + }, + { + "extract_ms": 5925.236199982464, + "serialize_ms": 0.6036998238414526, + "total_ms": 5925.839899806306 + } + ], + "repeated_median": { + "extract_ms": 2381.4179999753833, + "serialize_ms": 0.5778998602181673, + "total_ms": 2381.9958998356014 + }, + "peak_rss_bytes": 137129984, + "output_bytes": 1846, + "profile": { + "all_features": false, + "extract_ms": 2264.473000075668, + "serialize_ms": 0.32570003531873226, + "total_ms": 2264.798700110987, + "stages": { + "cell_extraction": { + "ms": 4.219199996441603, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.06960006430745125, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0055998098105192184, + "calls": 1 + }, + "table_detection": { + "ms": 136.24999998137355, + "calls": 1 + }, + "rich_extraction": { + "ms": 31.944999936968088, + "calls": 2 + }, + "model_construction": { + "ms": 0.0662999227643013, + "calls": 1 + }, + "workbook_parsing": { + "ms": 3.970199963077903, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "d581419f5c4c92d17f0680515c85cc175325852ebc16c495258619926e1a2d43", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:07:35.835537+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "style-heavy.xlsx", + "sha256": "05a77cc146594f2db73cb3aa764ad2625fa798809fce82fbe42900b5cc49529e", + "size_bytes": 38626 + }, + "mode": "light", + "startup": { + "python_process_ms": 49.910700181499124, + "package_import_process_ms": 124.31650003418326, + "cli_help_process_ms": 152.49649994075298, + "cold_extraction_process_ms": 1109.2504998669028, + "cold_extraction_in_process_ms": 881.8645998835564, + "measurement_worker_process_ms": 2333.5046998690814 + }, + "import_ms": 10.29270002618432, + "modules_after_import": [], + "modules_after_extraction": [ + "numpy", + "scipy", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 1306.6002998966724, + "serialize_ms": 17.962400102987885, + "total_ms": 1324.5626999996603 + }, + "repeated": [ + { + "extract_ms": 183.62839985638857, + "serialize_ms": 9.188500000163913, + "total_ms": 192.81689985655248 + }, + { + "extract_ms": 146.50429994799197, + "serialize_ms": 9.619200136512518, + "total_ms": 156.12350008450449 + }, + { + "extract_ms": 169.88579998724163, + "serialize_ms": 8.51630000397563, + "total_ms": 178.40209999121726 + } + ], + "repeated_median": { + "extract_ms": 169.88579998724163, + "serialize_ms": 9.188500000163913, + "total_ms": 178.40209999121726 + }, + "peak_rss_bytes": 77516800, + "output_bytes": 73844, + "profile": { + "all_features": false, + "extract_ms": 96.94099985063076, + "serialize_ms": 7.804099936038256, + "total_ms": 104.74509978666902, + "stages": { + "cell_extraction": { + "ms": 65.83840004168451, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.07399986498057842, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 29.875099891796708, + "calls": 1 + }, + "rich_extraction": { + "ms": 0.687299994751811, + "calls": 2 + }, + "model_construction": { + "ms": 0.08810008876025677, + "calls": 1 + }, + "workbook_parsing": { + "ms": 58.99429996497929, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 2, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "d15601bca18dbc97a66883ec4f81fc3ea8ec113849a746abdf97cb7db4a943f3", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:07:59.468980+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "style-heavy.xlsx", + "sha256": "05a77cc146594f2db73cb3aa764ad2625fa798809fce82fbe42900b5cc49529e", + "size_bytes": 38626 + }, + "mode": "standard", + "startup": { + "python_process_ms": 49.81900006532669, + "package_import_process_ms": 96.04030009359121, + "cli_help_process_ms": 104.20860000886023, + "cold_extraction_process_ms": 5224.18489982374, + "cold_extraction_in_process_ms": 4160.238399868831, + "measurement_worker_process_ms": 14286.10929986462 + }, + "import_ms": 5.810899892821908, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 4250.004000030458, + "serialize_ms": 11.30770007148385, + "total_ms": 4261.311700101942 + }, + "repeated": [ + { + "extract_ms": 2419.2167001310736, + "serialize_ms": 8.054499980062246, + "total_ms": 2427.271200111136 + }, + { + "extract_ms": 1861.7253999691457, + "serialize_ms": 8.458200143650174, + "total_ms": 1870.1836001127958 + }, + { + "extract_ms": 1993.3943999931216, + "serialize_ms": 9.072700049728155, + "total_ms": 2002.4671000428498 + } + ], + "repeated_median": { + "extract_ms": 1993.3943999931216, + "serialize_ms": 8.458200143650174, + "total_ms": 2002.4671000428498 + }, + "peak_rss_bytes": 138383360, + "output_bytes": 73844, + "profile": { + "all_features": false, + "extract_ms": 2410.3761999867857, + "serialize_ms": 9.171100100502372, + "total_ms": 2419.547300087288, + "stages": { + "cell_extraction": { + "ms": 141.62580017000437, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.09850016795098782, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.007899943739175797, + "calls": 1 + }, + "table_detection": { + "ms": 35.54870001971722, + "calls": 1 + }, + "rich_extraction": { + "ms": 22.741899825632572, + "calls": 2 + }, + "model_construction": { + "ms": 0.09660003706812859, + "calls": 1 + }, + "workbook_parsing": { + "ms": 133.57830001041293, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "d15601bca18dbc97a66883ec4f81fc3ea8ec113849a746abdf97cb7db4a943f3", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:08:12.691158+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "table-heavy.xlsx", + "sha256": "a319c15ab9b4987527beaa627ca336a679e9091f34427c08710199cfda776e5f", + "size_bytes": 38745 + }, + "mode": "light", + "startup": { + "python_process_ms": 40.53639993071556, + "package_import_process_ms": 107.6005999930203, + "cli_help_process_ms": 229.41979998722672, + "cold_extraction_process_ms": 1166.3363999687135, + "cold_extraction_in_process_ms": 962.1197001542896, + "measurement_worker_process_ms": 1540.8386001363397 + }, + "import_ms": 7.586700143292546, + "modules_after_import": [], + "modules_after_extraction": [ + "numpy", + "scipy", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 835.1821999531239, + "serialize_ms": 12.88280007429421, + "total_ms": 848.0650000274181 + }, + "repeated": [ + { + "extract_ms": 60.73109991848469, + "serialize_ms": 11.076699942350388, + "total_ms": 71.80779986083508 + }, + { + "extract_ms": 100.02809995785356, + "serialize_ms": 12.845800025388598, + "total_ms": 112.87389998324215 + }, + { + "extract_ms": 61.34120002388954, + "serialize_ms": 10.516899870708585, + "total_ms": 71.85809989459813 + } + ], + "repeated_median": { + "extract_ms": 61.34120002388954, + "serialize_ms": 11.076699942350388, + "total_ms": 71.85809989459813 + }, + "peak_rss_bytes": 78110720, + "output_bytes": 92326, + "profile": { + "all_features": false, + "extract_ms": 89.98739998787642, + "serialize_ms": 10.007600067183375, + "total_ms": 99.99500005505979, + "stages": { + "cell_extraction": { + "ms": 81.88840001821518, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.11849985457956791, + "calls": 1 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.0, + "calls": 0 + }, + "table_detection": { + "ms": 6.563499802723527, + "calls": 1 + }, + "rich_extraction": { + "ms": 1.0501001961529255, + "calls": 2 + }, + "model_construction": { + "ms": 0.08059991523623466, + "calls": 1 + }, + "workbook_parsing": { + "ms": 72.3176000174135, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 2, + "pipeline_state": { + "com_attempted": false, + "com_succeeded": false, + "fallback_reason": null + } + }, + "output_sha256": "46649d4d0baeb567b1bf589ae5cd294688787938cb695f1c8802efd9d50282d7", + "before_after_serialized_output_equal": true + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:08:36.106916+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "table-heavy.xlsx", + "sha256": "a319c15ab9b4987527beaa627ca336a679e9091f34427c08710199cfda776e5f", + "size_bytes": 38745 + }, + "mode": "standard", + "startup": { + "python_process_ms": 66.25989987514913, + "package_import_process_ms": 107.71030001342297, + "cli_help_process_ms": 138.61409993842244, + "cold_extraction_process_ms": 3958.7105999235064, + "cold_extraction_in_process_ms": 3191.063900012523, + "measurement_worker_process_ms": 14385.283499956131 + }, + "import_ms": 6.533199921250343, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 4616.012000013143, + "serialize_ms": 15.782899921759963, + "total_ms": 4631.794899934903 + }, + "repeated": [ + { + "extract_ms": 2413.535499945283, + "serialize_ms": 11.348600033670664, + "total_ms": 2424.8840999789536 + }, + { + "extract_ms": 2000.9305998682976, + "serialize_ms": 11.404700111597776, + "total_ms": 2012.3352999798954 + }, + { + "extract_ms": 2244.873399846256, + "serialize_ms": 12.066699797287583, + "total_ms": 2256.9400996435434 + } + ], + "repeated_median": { + "extract_ms": 2244.873399846256, + "serialize_ms": 11.404700111597776, + "total_ms": 2256.9400996435434 + }, + "peak_rss_bytes": 138334208, + "output_bytes": 92326, + "profile": { + "all_features": false, + "extract_ms": 1963.5684001259506, + "serialize_ms": 11.708399979397655, + "total_ms": 1975.2768001053482, + "stages": { + "cell_extraction": { + "ms": 60.241700150072575, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.15139998868107796, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.011599855497479439, + "calls": 1 + }, + "table_detection": { + "ms": 11.130300117656589, + "calls": 1 + }, + "rich_extraction": { + "ms": 28.78309995867312, + "calls": 2 + }, + "model_construction": { + "ms": 0.07329997606575489, + "calls": 1 + }, + "workbook_parsing": { + "ms": 50.2099001314491, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + }, + "output_sha256": "46649d4d0baeb567b1bf589ae5cd294688787938cb695f1c8802efd9d50282d7", + "before_after_serialized_output_equal": true + } + ] +} diff --git a/benchmark/baselines/issues145-148-standard-recheck-2026-10-03-windows.json b/benchmark/baselines/issues145-148-standard-recheck-2026-10-03-windows.json new file mode 100644 index 00000000..2ced38a1 --- /dev/null +++ b/benchmark/baselines/issues145-148-standard-recheck-2026-10-03-windows.json @@ -0,0 +1,562 @@ +{ + "schema_version": 1, + "reason": "Initial standard-mode small/sparse observations regressed while initial integrated pass improved. Repeat both sources in one serial slot; retain both datasets rather than replacing unfavorable observations.", + "results": [ + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T06:09:44.740726+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "small.xlsx", + "sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "size_bytes": 7221 + }, + "mode": "standard", + "startup": { + "python_process_ms": 47.087799990549684, + "package_import_process_ms": 96.15039988420904, + "cli_help_process_ms": 100.53570009768009, + "cold_extraction_process_ms": 4645.027499878779, + "cold_extraction_in_process_ms": 3763.1107000634074, + "measurement_worker_process_ms": 16940.328900003806 + }, + "import_ms": 6.2193002086132765, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3152.787300059572, + "serialize_ms": 2.8085000813007355, + "total_ms": 3155.5958001408726 + }, + "repeated": [ + { + "extract_ms": 1884.360000025481, + "serialize_ms": 0.6321999244391918, + "total_ms": 1884.9921999499202 + }, + { + "extract_ms": 1803.438700037077, + "serialize_ms": 0.6315000355243683, + "total_ms": 1804.0702000726014 + }, + { + "extract_ms": 1825.2989000175148, + "serialize_ms": 0.5033998750150204, + "total_ms": 1825.8022998925298 + }, + { + "extract_ms": 2120.2287001069635, + "serialize_ms": 0.694700051099062, + "total_ms": 2120.9234001580626 + }, + { + "extract_ms": 2449.3233000393957, + "serialize_ms": 0.5830000154674053, + "total_ms": 2449.906300054863 + } + ], + "repeated_median": { + "extract_ms": 1884.360000025481, + "serialize_ms": 0.6315000355243683, + "total_ms": 1884.9921999499202 + }, + "peak_rss_bytes": 128589824, + "output_bytes": 2422, + "profile": { + "all_features": false, + "extract_ms": 2690.2729000430554, + "serialize_ms": 0.6183998193591833, + "total_ms": 2690.8912998624146, + "stages": { + "cell_extraction": { + "ms": 7.631100015714765, + "calls": 1 + }, + "print_area_extraction": { + "ms": 5.247800145298243, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 4.968900000676513, + "calls": 1 + }, + "table_detection": { + "ms": 13.281800085678697, + "calls": 1 + }, + "rich_extraction": { + "ms": 52.36550001427531, + "calls": 2 + }, + "model_construction": { + "ms": 0.06909994408488274, + "calls": 1 + }, + "workbook_parsing": { + "ms": 23.94119999371469, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + } + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:10:04.423894+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "small.xlsx", + "sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "size_bytes": 7221 + }, + "mode": "standard", + "startup": { + "python_process_ms": 44.50630000792444, + "package_import_process_ms": 92.57240011356771, + "cli_help_process_ms": 126.30530004389584, + "cold_extraction_process_ms": 3802.241099998355, + "cold_extraction_in_process_ms": 2978.772500064224, + "measurement_worker_process_ms": 15323.599900119007 + }, + "import_ms": 6.799299968406558, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3184.673500014469, + "serialize_ms": 2.872099867090583, + "total_ms": 3187.5455998815596 + }, + "repeated": [ + { + "extract_ms": 2060.647599864751, + "serialize_ms": 0.45359996147453785, + "total_ms": 2061.1011998262256 + }, + { + "extract_ms": 1778.2715000212193, + "serialize_ms": 0.6323999259620905, + "total_ms": 1778.9038999471813 + }, + { + "extract_ms": 1893.8827000092715, + "serialize_ms": 0.6173998117446899, + "total_ms": 1894.5000998210162 + }, + { + "extract_ms": 1850.7412001490593, + "serialize_ms": 0.5727000534534454, + "total_ms": 1851.3139002025127 + }, + { + "extract_ms": 1928.5403001122177, + "serialize_ms": 0.4547999706119299, + "total_ms": 1928.9951000828296 + } + ], + "repeated_median": { + "extract_ms": 1893.8827000092715, + "serialize_ms": 0.5727000534534454, + "total_ms": 1894.5000998210162 + }, + "peak_rss_bytes": 126844928, + "output_bytes": 2422, + "profile": { + "all_features": false, + "extract_ms": 1787.805600091815, + "serialize_ms": 0.6766000296920538, + "total_ms": 1788.482200121507, + "stages": { + "cell_extraction": { + "ms": 4.365700064226985, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.04910002462565899, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.012400094419717789, + "calls": 1 + }, + "table_detection": { + "ms": 2.662800019606948, + "calls": 1 + }, + "rich_extraction": { + "ms": 39.007899817079306, + "calls": 2 + }, + "model_construction": { + "ms": 0.07039983756840229, + "calls": 1 + }, + "workbook_parsing": { + "ms": 4.116100026294589, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + } + }, + { + "comparison": "before", + "schema_version": 1, + "recorded_at": "2026-10-03T06:10:25.576246+00:00", + "source": { + "revision": "5bcb0e47984104414956c9232bfca75b5076ddec", + "dirty": false + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "sparse.xlsx", + "sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "size_bytes": 5417 + }, + "mode": "standard", + "startup": { + "python_process_ms": 36.9329999666661, + "package_import_process_ms": 85.47840011306107, + "cli_help_process_ms": 102.73239994421601, + "cold_extraction_process_ms": 4365.64249987714, + "cold_extraction_in_process_ms": 3443.148500053212, + "measurement_worker_process_ms": 16278.160999994725 + }, + "import_ms": 5.09929982945323, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 2867.8792000282556, + "serialize_ms": 3.203399945050478, + "total_ms": 2871.082599973306 + }, + "repeated": [ + { + "extract_ms": 1796.610799850896, + "serialize_ms": 0.3690000157803297, + "total_ms": 1796.9797998666763 + }, + { + "extract_ms": 2120.6265001092106, + "serialize_ms": 0.421800184994936, + "total_ms": 2121.0483002942055 + }, + { + "extract_ms": 1962.1656001545489, + "serialize_ms": 0.4438001196831465, + "total_ms": 1962.609400274232 + }, + { + "extract_ms": 2140.074700117111, + "serialize_ms": 0.32139988616108894, + "total_ms": 2140.3961000032723 + }, + { + "extract_ms": 2182.3250001762062, + "serialize_ms": 0.42010005563497543, + "total_ms": 2182.745100231841 + } + ], + "repeated_median": { + "extract_ms": 2120.6265001092106, + "serialize_ms": 0.42010005563497543, + "total_ms": 2121.0483002942055 + }, + "peak_rss_bytes": 139956224, + "output_bytes": 1846, + "profile": { + "all_features": false, + "extract_ms": 2305.021299980581, + "serialize_ms": 0.32130000181496143, + "total_ms": 2305.3425999823958, + "stages": { + "cell_extraction": { + "ms": 22.496999939903617, + "calls": 1 + }, + "print_area_extraction": { + "ms": 4.313600016757846, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 3.5459999926388264, + "calls": 1 + }, + "table_detection": { + "ms": 152.60799997486174, + "calls": 1 + }, + "rich_extraction": { + "ms": 25.457600131630898, + "calls": 2 + }, + "model_construction": { + "ms": 0.060499878600239754, + "calls": 1 + }, + "workbook_parsing": { + "ms": 17.94409961439669, + "calls": 5 + } + }, + "openpyxl_workbook_opens": 5, + "archive_opens": 6, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + } + }, + { + "comparison": "after", + "schema_version": 1, + "recorded_at": "2026-10-03T06:10:50.026370+00:00", + "source": { + "revision": "10f53cf13e30fcec88b33f3dbc4f2c41118690a5", + "dirty": true + }, + "environment": { + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "executable": "", + "platform": "Windows-10-10.0.22631-SP0", + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "pandas": "3.0.6", + "numpy": "2.4.6", + "scipy": "1.17.1", + "xlwings": "0.37.4", + "pydantic": "2.13.5" + }, + "SKIP_COM_TESTS": null, + "EXSTRUCT_BORDER_CLUSTER_BACKEND": null + }, + "input": { + "name": "sparse.xlsx", + "sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "size_bytes": 5417 + }, + "mode": "standard", + "startup": { + "python_process_ms": 57.15379980392754, + "package_import_process_ms": 109.80409989133477, + "cli_help_process_ms": 114.56699995324016, + "cold_extraction_process_ms": 4116.35209992528, + "cold_extraction_in_process_ms": 3244.369300082326, + "measurement_worker_process_ms": 19748.15639993176 + }, + "import_ms": 7.442299975082278, + "modules_after_import": [], + "modules_after_extraction": [ + "pandas", + "numpy", + "scipy", + "xlwings", + "openpyxl", + "PIL" + ], + "first": { + "extract_ms": 3575.400800211355, + "serialize_ms": 4.273200174793601, + "total_ms": 3579.6740003861487 + }, + "repeated": [ + { + "extract_ms": 4373.783599818125, + "serialize_ms": 0.4091998562216759, + "total_ms": 4374.192799674347 + }, + { + "extract_ms": 2051.175800152123, + "serialize_ms": 0.3551999107003212, + "total_ms": 2051.5310000628233 + }, + { + "extract_ms": 2165.97339999862, + "serialize_ms": 0.38049998693168163, + "total_ms": 2166.353899985552 + }, + { + "extract_ms": 2206.7909000907093, + "serialize_ms": 0.39590010419487953, + "total_ms": 2207.186800194904 + }, + { + "extract_ms": 2358.3802999928594, + "serialize_ms": 0.38359989412128925, + "total_ms": 2358.7638998869807 + } + ], + "repeated_median": { + "extract_ms": 2206.7909000907093, + "serialize_ms": 0.38359989412128925, + "total_ms": 2207.186800194904 + }, + "peak_rss_bytes": 136998912, + "output_bytes": 1846, + "profile": { + "all_features": false, + "extract_ms": 2021.5338999405503, + "serialize_ms": 0.37480005994439125, + "total_ms": 2021.9087000004947, + "stages": { + "cell_extraction": { + "ms": 4.4340998865664005, + "calls": 1 + }, + "print_area_extraction": { + "ms": 0.07889978587627411, + "calls": 2 + }, + "formula_extraction": { + "ms": 0.0, + "calls": 0 + }, + "color_extraction": { + "ms": 0.0, + "calls": 0 + }, + "merged_cell_extraction": { + "ms": 0.006100162863731384, + "calls": 1 + }, + "table_detection": { + "ms": 124.6800001244992, + "calls": 1 + }, + "rich_extraction": { + "ms": 34.88779999315739, + "calls": 2 + }, + "model_construction": { + "ms": 0.06260001100599766, + "calls": 1 + }, + "workbook_parsing": { + "ms": 4.199799848720431, + "calls": 1 + } + }, + "openpyxl_workbook_opens": 1, + "archive_opens": 1, + "pipeline_state": { + "com_attempted": true, + "com_succeeded": true, + "fallback_reason": null + } + } + } + ] +} diff --git a/benchmark/issue148-results.md b/benchmark/issue148-results.md new file mode 100644 index 00000000..2f8c985d --- /dev/null +++ b/benchmark/issue148-results.md @@ -0,0 +1,76 @@ +# Issue 148 clustering evaluation + +SciPy is now optional through `exstruct[fast]` (also included in `[all]`). +The user selected this policy after reviewing the serial Windows measurements +below on 2026-10-03. NumPy remains mandatory. Existing `auto` / `python` / `numpy` +selection and accelerated-exception fallback are unchanged. Sparse-set BFS is an +evaluation-only candidate and is not selected in production. + +## Measurements and correctness + +`baselines/issue148-2026-10-03-windows.json` records one warmup and seven timed +repeats on Windows, Python 3.11.13, NumPy 2.4.6, SciPy 1.17.1 and openpyxl 3.1.5. +Source was integrated revision `49f2d66` with documentation/result changes pending. +Timing excludes fixture generation; clustering includes grid-to-set conversion. +Every backend produced identical bounding boxes, ordering and full light JSON. + +| Workload | Grid / occupied cells | Clustering median ms: SciPy / Python / candidate | Light extraction + JSON median ms: SciPy / Python / candidate | +| --- | --- | --- | --- | +| sparse-small | 40 × 20 / 44 | 0.031 / 0.077 / 0.031 | 8.8 / 9.6 / 8.5 | +| sparse-large | 2000 × 200 / 74 | 2.511 / 18.658 / 0.618 | 1150 / 1076 / 1099 | +| dense-style-heavy | 300 × 80 / 24000 | 0.217 / 19.959 / 26.997 | 529 / 590 / 524 | +| many-disconnected | 300 × 100 / 4000 | 53.113 / 4.877 / 3.049 | 374 / 299 / 322 | + +Dense labeling strongly favors SciPy, while many separated components favor +Python. The candidate improves some isolated clustering cases but does not +establish consistent whole-extraction gains; it is not promoted. These synthetic +sequential observations are not universal performance targets. No RSS, +process-cold or Excel COM measurement is claimed by this comparison. The general +before/after runner measures those separately. + +Raw samples, dependency versions, source state, environment, grid/fixture hashes, +component counts and output hashes/bytes are in JSON. The earlier untimed +`baselines/issue148-2026-10-03-correctness.json` is retained as characterization +evidence; its empty timing dictionaries mean untimed rather than zero latency. +Inputs have fixed ZIP/core timestamps. Style-heavy workbooks include borders, +fills, fonts and number formats. Extent markers and normal scan limits apply. +These timings precede the integrated sparse worksheet reader refinement; they +support dependency selection, not final before/after extraction speed claims. + +## Reproduce + +```powershell +rtk proxy uv sync --locked --all-extras +rtk proxy uv run --no-sync python -m benchmark.issue148_clustering --check-only --output output/issue148-correctness.json +# Run in an exclusive measurement slot, without other tests or Excel jobs: +rtk proxy uv run --no-sync python -m benchmark.issue148_clustering --repeats 7 --output output/issue148-timed.json +``` + +The reference comparison requires SciPy. Missing optional packages and unavailable +Git metadata are recorded as null. Full extraction temporarily patches the live +Python clustering function and restores overrides/environment even on failure. +`candidate_sparse` is an evaluation label, not a supported environment value. + +## Base installed-package verification + +The base-install CI jobs install the package without development or acceleration +extras, then run `python -m benchmark.issue148_scipy_free --installed`. This smoke +requires both pandas and SciPy to be absent, refuses source-checkout imports and +compares actual bordered-workbook light JSON for `python`, `auto` and `numpy`. +For `auto` and `numpy`, the existing accelerated failure reaches Python fallback. +Fresh subprocess tests also block SciPy before importing ExStruct and cover +accelerated execution exceptions and live helper overrides. + +For a local wheel check: + +```powershell +rtk proxy uv build --wheel --out-dir output/base-wheel +rtk proxy uv venv output/base-install --python .venv/Scripts/python.exe +rtk proxy uv pip install --python output/base-install/Scripts/python.exe output/base-wheel/exstruct-0.8.2-py3-none-any.whl +rtk proxy output/base-install/Scripts/python.exe -m benchmark.issue148_scipy_free --installed +``` + +The environment is separate from the normal development environment, which +includes pandas for characterization and SciPy when installing `fast`/`all`. +See ADR-0012 for the dependency-policy rationale and `docs/api.md` for the +legacy `numpy` name's SciPy-backed meaning. diff --git a/benchmark/issue148_clustering.py b/benchmark/issue148_clustering.py new file mode 100644 index 00000000..1b81b7b8 --- /dev/null +++ b/benchmark/issue148_clustering.py @@ -0,0 +1,221 @@ +"""Issue 148 comparison; timings require a coordinator's serial slot. + +Run correctness only: python -m benchmark.issue148_clustering --check-only +Run measurements: python -m benchmark.issue148_clustering --repeats 7 --output ... +All maps include grid-to-set construction in candidate timing. End-to-end +measurements include light extraction plus default JSON serialization, with +only the live clustering implementation overridden and restored afterwards. +""" + +import argparse +from collections.abc import Callable +from datetime import UTC, datetime +from functools import partial +import hashlib +from importlib.metadata import PackageNotFoundError, version +import json +import os +from pathlib import Path +import platform +from statistics import median +import subprocess +import sys +from tempfile import TemporaryDirectory +from time import perf_counter +from typing import Any +from unittest.mock import patch + +import numpy as np +from openpyxl import Workbook +from openpyxl.styles import Border, Font, PatternFill, Side + +import exstruct +from exstruct.core import cells + +from .performance_fixtures import _normalize_archive + +Cluster = Callable[[np.ndarray, int], list[tuple[int, int, int, int]]] +IMPLEMENTATIONS: dict[str, Cluster] = { + "scipy": cells._detect_border_clusters_numpy, + "python": cells._detect_border_clusters_python, + "candidate_sparse": cells._detect_border_clusters_sparse, +} + + +def border_maps() -> dict[str, np.ndarray]: + """Fixed occupied regions, including a large sparse grid and many labels.""" + small = np.zeros((40, 20), dtype=bool) + small[2:8, 1:5] = True + small[25:29, 12:17] = True + large = np.zeros((2000, 200), dtype=bool) + large[2:8, 1:5] = True + large[1800:1805, 170:180] = True + dense = np.ones((300, 80), dtype=bool) + disconnected = np.zeros((300, 100), dtype=bool) + for row in range(0, 300, 6): + for col in range(0, 100, 5): + disconnected[row : row + 2, col : col + 2] = True + return { + "sparse-small": small, + "sparse-large": large, + "dense-style-heavy": dense, + "many-disconnected": disconnected, + } + + +def write_fixture(path: Path, grid: np.ndarray) -> None: + """Reproducible styled workbook preserving the complete input dimensions.""" + workbook = Workbook() + workbook.properties.created = datetime(2000, 1, 1) + workbook.properties.modified = datetime(2000, 1, 1) + sheet = workbook.active + assert sheet is not None + border = Border(bottom=Side(style="thin")) + rows, cols = np.nonzero(grid) + for row, col in zip(rows.tolist(), cols.tolist(), strict=True): + cell = sheet.cell(row + 1, col + 1, f"r{row}c{col}") + cell.border = border + if path.stem == "dense-style-heavy": + cell.fill = PatternFill("solid", fgColor=f"{row * 313 % 0xFFFFFF:06X}") + cell.font = Font(bold=row % 2 == 0, size=9 + col % 6) + cell.number_format = "0.00" + sheet.cell(grid.shape[0], grid.shape[1], "extent") + workbook.save(path) + workbook.close() + _normalize_archive(path) + + +def observations( + operation: Callable[[], Any], repeats: int, check_only: bool +) -> tuple[Any, dict[str, Any]]: + """Warm correctness call followed by raw warm samples; no cold claim.""" + reference = operation() + if check_only: + return reference, {} + samples: list[float] = [] + for _ in range(repeats): + start = perf_counter() + result = operation() + samples.append((perf_counter() - start) * 1000) + if result != reference: + raise AssertionError("Output changed between measurement repetitions") + return reference, {"warm_ms": samples, "median_ms": median(samples)} + + +def extraction(path: Path, implementation: Cluster) -> str: + """Use existing python routing with a temporary live implementation patch.""" + with ( + patch.dict(os.environ, {"EXSTRUCT_BORDER_CLUSTER_BACKEND": "python"}), + patch.object(cells, "_detect_border_clusters_python", implementation), + ): + return exstruct.serialize_workbook( + exstruct.extract(path, mode="light"), fmt="json" + ) + + +def run_comparison(repeats: int, check_only: bool) -> dict[str, Any]: + """Assert bbox and entire serialized output equality across all backends.""" + results: dict[str, Any] = {} + with TemporaryDirectory(prefix="issue148-") as temporary: + for category, grid in border_maps().items(): + path = Path(temporary) / f"{category}.xlsx" + write_fixture(path, grid) + case: dict[str, Any] = { + "shape": list(grid.shape), + "occupied": int(np.count_nonzero(grid)), + "grid_bytes": grid.nbytes, + "grid_sha256": hashlib.sha256(grid.tobytes()).hexdigest(), + "fixture_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "min_size": 4, + "cluster": {}, + "light_extraction_serialization": {}, + } + expected_boxes = None + expected_output = None + for name, implementation in IMPLEMENTATIONS.items(): + boxes, timings = observations( + partial(implementation, grid, 4), repeats, check_only + ) + if expected_boxes is None: + expected_boxes = boxes + assert boxes == expected_boxes, (category, name, "bbox/order mismatch") + case["cluster"][name] = timings + serialized, timings = observations( + partial(extraction, path, implementation), repeats, check_only + ) + if expected_output is None: + expected_output = serialized + assert serialized == expected_output, ( + category, + name, + "extraction mismatch", + ) + case["light_extraction_serialization"][name] = timings + assert expected_boxes is not None and expected_output is not None + case["components"] = len(expected_boxes) + case["output_sha256"] = hashlib.sha256(expected_output.encode()).hexdigest() + case["output_bytes"] = len(expected_output.encode()) + case["bbox_order_equal"] = case["serialized_output_equal"] = True + results[category] = case + return results + + +def git_metadata(arguments: list[str]) -> str | None: + try: + result = subprocess.run( + ["git", *arguments], capture_output=True, text=True, timeout=10, check=False + ) + except (OSError, subprocess.TimeoutExpired): + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def dependency_version(name: str) -> str | None: + """Record optional absent dependencies without preventing comparison.""" + try: + return version(name) + except PackageNotFoundError: + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.repeats < 1: + parser.error("--repeats must be positive") + status = git_metadata(["status", "--porcelain"]) + result = { + "schema_version": 1, + "timestamp_utc": datetime.now(UTC).isoformat(), + "source_revision": git_metadata(["rev-parse", "HEAD"]), + "source_dirty": None if status is None else bool(status), + "python": sys.version, + "platform": platform.platform(), + "dependencies": { + name: dependency_version(name) + for name in ("numpy", "scipy", "pandas", "openpyxl") + }, + "environment": { + key: value + for key, value in os.environ.items() + if key.startswith("EXSTRUCT_") + }, + "mode": "light", + "warm_repeats": 0 if args.check_only else args.repeats, + "check_only": args.check_only, + "cases": run_comparison(args.repeats, args.check_only), + } + output = json.dumps(result, indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(output, encoding="utf-8") + else: + print(output, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/issue148_scipy_free.py b/benchmark/issue148_scipy_free.py new file mode 100644 index 00000000..cecfd899 --- /dev/null +++ b/benchmark/issue148_scipy_free.py @@ -0,0 +1,80 @@ +"""Untimed runtime smoke in a genuinely SciPy-free base installation. + +Install the base wheel in a clean environment and invoke this module with +--installed to verify packaging as well as runtime behavior. Without that flag, +the module selects the source checkout for development comparisons. +""" + +import importlib.util +import argparse +import json +import os +from pathlib import Path +import sys +from tempfile import TemporaryDirectory +from unittest.mock import patch + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--installed", action="store_true") + args = parser.parse_args() + if importlib.util.find_spec("scipy") is not None: + raise RuntimeError("This smoke requires an environment without SciPy") + if importlib.util.find_spec("pandas") is not None: + raise RuntimeError("This smoke requires an environment without pandas") + if not args.installed: + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + import numpy as np + from openpyxl import Workbook + from openpyxl.styles import Border, Side + + import exstruct + + if args.installed and Path(exstruct.__file__).resolve().is_relative_to( + Path(__file__).resolve().parents[1] / "src" + ): + raise RuntimeError("Installed smoke unexpectedly imported source checkout") + from exstruct.core import cells + + expected = None + with TemporaryDirectory(prefix="issue148-base-") as directory: + path = Path(directory) / "bordered.xlsx" + workbook = Workbook() + sheet = workbook.active + assert sheet is not None + border = Border(bottom=Side(style="thin")) + for row in range(1, 5): + for col in range(1, 4): + sheet.cell(row, col, f"value-{row}-{col}").border = border + workbook.save(path) + workbook.close() + for backend in ("python", "auto", "numpy"): + with patch.dict(os.environ, {"EXSTRUCT_BORDER_CLUSTER_BACKEND": backend}): + assert cells.detect_border_clusters(np.ones((2, 2), dtype=bool)) == [ + (0, 0, 1, 1) + ] + result = exstruct.extract(path, mode="light") + assert result.sheets["Sheet"].rows + assert result.sheets["Sheet"].table_candidates + serialized = exstruct.serialize_workbook(result, fmt="json") + if expected is None: + expected = serialized + assert serialized == expected + assert not any(name == "scipy" or name.startswith("scipy.") for name in sys.modules) + print( + json.dumps( + { + "scipy_installed": False, + "pandas_installed": False, + "installed_package": args.installed, + "backends": ["python", "auto", "numpy"], + "light_output_equal": True, + } + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/issues145-148-results.md b/benchmark/issues145-148-results.md new file mode 100644 index 00000000..5b008816 --- /dev/null +++ b/benchmark/issues145-148-results.md @@ -0,0 +1,93 @@ +# Issues 145-148 integrated extraction measurements + +## Method and compatibility + +`baselines/issues145-148-2026-10-03-windows.json` compares base `5bcb0e4` and +integrated extraction code `10f53cf` using the same locked Windows Python 3.11.13 +interpreter and six deterministic fixture hashes. Baseline observations were +collected first; final after observations were rerun following the sparse-reader +refinement. Each run has one first observation and three warm repeats, plus +separate fresh-process import/CLI/cold extraction probes and an instrumented +profile. Operations run serially. Source `dirty=true` includes pending +measurement artifacts and smoke-script formatting; production extraction code +is committed at the recorded revision. Local interpreter paths are redacted. + +Independent extractions compared exact compact JSON bytes before and after for +all twelve input/mode combinations; every output hash matched. All standard +profiles recorded COM attempted and succeeded with no fallback. Default profiles +load one shared openpyxl workbook after integration; the old implementation +loaded four in light and five in standard, rising to 50/51 for 24 sheets. +Formula-enabled tests require two variants independently of sheet count. +Openpyxl loader counts exclude Excel COM and OOXML drawing ZIP reads. + +## Observations + +Warm total includes extraction and JSON serialization. Cold total is fresh child +wall time including startup, imports and teardown. RSS is Python lifetime peak +working set after uninstrumented runs, excluding Excel child processes and later +instrumentation. Times are ms and RSS is MiB. + +| Input | Mode | Warm before / after | Cold before / after | Peak RSS before / after | +| --- | --- | ---: | ---: | ---: | +| large | light | 1595.9 / 596.3 | 3224 / 1612 | 189.1 / 109.3 | +| large | standard | 5605.5 / 2495.0 | 5848 / 4126 | 187.6 / 165.9 | +| many-sheet | light | 4101.8 / 119.5 | 5120 / 1030 | 133.2 / 75.6 | +| many-sheet | standard | 6133.0 / 2485.3 | 8134 / 4638 | 137.8 / 129.3 | +| small | light | 23.1 / 7.4 | 1553 / 851 | 116.6 / 64.3 | +| small | standard | 1961.2 / 2500.5 | 3359 / 3783 | 122.1 / 121.0 | +| sparse | light | 70.2 / 41.4 | 2173 / 1338 | 126.2 / 71.1 | +| sparse | standard | 1660.3 / 2382.0 | 3716 / 4639 | 130.8 / 130.8 | +| style-heavy | light | 535.4 / 178.4 | 1905 / 1109 | 128.1 / 73.9 | +| style-heavy | standard | 1907.8 / 2002.5 | 4322 / 5224 | 135.3 / 132.0 | +| table-heavy | light | 242.1 / 71.9 | 2301 / 1166 | 129.9 / 74.5 | +| table-heavy | standard | 2325.3 / 2256.9 | 3800 / 3959 | 136.3 / 131.9 | + +All six light workloads improved warm/cold times and Python peak RSS in this +run. COM dominates smaller standard workloads; the initial integrated pass +improved small/sparse, whereas the final pass above regressed. Neither pass +establishes a universal standard-mode improvement. A paired five-repeat recheck +is retained in `baselines/issues145-148-standard-recheck-2026-10-03-windows.json` +so unfavorable observations remain visible. No strict shared-runner time gate +or broad statistical claim is made. + +The paired recheck measured small standard at 1885.0 / 1894.5 ms (before/after) +and sparse standard at 2121.0 / 2207.2 ms, with COM success throughout. This +reduces the apparent regression relative to the earlier separated observations; +the remaining differences must still be read as local timings, not a guarantee. + +The first integrated sparse worksheet measurement regressed from 70 to 154 ms +because regular `iter_rows()` materialized every blank in its rectangular +extent. The final reader scans sorted existing worksheet cell coordinates, +retaining row gaps, column keys and link semantics without inserting blanks. +The regression test asserts both exact output and unchanged sparse storage. +Read-only standalone readers continue to stream `iter_rows()` and reset declared +dimensions, preserving the previous reader's behavior for malformed bounds. + +## Verification + +- Final non-COM/non-render suite after the backend-alias review fix: 1017 passed, + 1 LibreOffice smoke skipped, 11 deselected; coverage 83.04%, exceeding 80%. +- Actual Excel COM tests: eight passed on this Windows host. Pywin32 emitted + RPC diagnostics during that run; successful process exit and test results + are distinct from those diagnostics. +- Actual forced LibreOffice smoke: one passed on this host. +- Actual PDF/PNG render suite: three passed. RPC diagnostics also appeared in + that run; output PDF/PNG assertions and the test process succeeded. +- Ruff, format and strict mypy passed; session tests cover close on + success/fallback/error, formula variants, helper overrides and concurrency. +- A built base wheel installed normally into an independent venv without pandas + or SciPy; bordered light JSON matched for `python`, `auto`, and `numpy`. + Actual BIFF light extraction also succeeded in that installed environment. +- A separate clean wheel install with `[fast]` installed SciPy and executed real + accelerated labeling successfully, while pandas remained absent. + +Timing datasets target `10f53cf`; subsequent `4952ed9` restores legacy backend +alias overrides and formats the installed smoke. Its full non-COM regression +run passed; the complete performance suite was not rerun for that narrow fix. +The subsequent Codacy follow-up uses static backend imports, explicit module +attribute resolution and normal session guards while retaining the same lazy +and override contracts. The final non-COM suite also passed after that refactor. + +SciPy comparison and the user-selected optional-dependency decision are in +`issue148-results.md` and ADR-0012. Sparse BFS remains evaluation-only. These +fixtures are synthetic probes rather than every production workbook. diff --git a/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md b/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md new file mode 100644 index 00000000..1ba90e27 --- /dev/null +++ b/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md @@ -0,0 +1,87 @@ +# ADR-0011: Extraction-Scoped Workbook Reuse and Direct Cell Reading + +## 状態 + +`proposed` + +## 背景 + +Issues #145-#147, under #142, address unnecessary DataFrame conversion, +repeated workbook parsing and backend imports on extraction startup. ExStruct +returns typed workbook data rather than DataFrames, so retaining pandas in the +production cell reader adds a substantial dependency without providing a +public abstraction. Independent path-based helpers also parse the same OOXML +workbook repeatedly, including from nested table detection helpers. + +ADR-0010 already defines light as a pure-Python rich OOXML baseline. This +decision improves its implementation while retaining that mode boundary, +serialization, coordinate conventions and fallback behavior. It does not +introduce a new public extraction backend or change COM-first/pre-COM ordering. + +## 決定 + +- Read `.xlsx` and `.xlsm` cells directly through openpyxl worksheet iteration; + build `CellRow` without a pandas DataFrame. Preserve the previous reader's + observable normalization, empty-cell filtering and cached-formula behavior. +- Retain `.xls` cell reading through a lazily imported direct xlrd reader. This + preserves the public input format without retaining pandas or requiring Excel + COM for cell reading. It does not add OOXML artifacts to BIFF workbooks. +- Scope openpyxl workbook ownership to one extraction invocation. Compatible + stages reuse a regular `data_only=True` workbook. Open a separate + `data_only=False` workbook only when openpyxl formula extraction is requested. +- Close owned workbooks when the extraction finishes or raises, including + fallback paths. Retain path-based standalone entrypoints with local ownership. + Do not cache workbooks globally or across extraction invocations. +- Keep common backend contracts independent of concrete COM implementations; + import the selected backend only when execution needs it. Light OOXML rich + extraction must retain shapes and charts while avoiding COM/render imports. +- Preserve compatibility call sites used for overrides and monkeypatches. Import + isolation must not change which overridden function an entrypoint executes. +- Evaluate clustering accelerators separately in #148. ADR-0012 records the + user-selected optional SciPy policy after measurements; this ADR does not + change clustering selection defaults. + +## 影響 + +- Direct cell construction removes production DataFrame allocation and the + pandas runtime requirement; pandas may remain in development/benchmark tools. +- Per-invocation ownership reduces parsing cost without stale data across calls + or cross-extraction resource sharing. Formula extraction still needs a second + workbook because its load semantics differ from cached cell values. +- A regular openpyxl workbook retains more resident objects than a streaming + read-only reader. Sharing it is preferred because borders, hyperlinks and + merged-cell extraction require those objects; memory and latency must still be + measured independently. +- Direct readers must explicitly preserve pandas-era behavior for values such + as NA tokens, booleans, errors and dates. Characterization regressions are + needed to prevent dependency removal from changing output semantics. +- xlrd becomes an explicit core dependency for the existing `.xls` input + capability, though it is imported only on that path. NumPy remains required + for current table detection; this change does not claim a dependency-free + light implementation. +- Lazy compatibility wrappers add indirection and require subprocess tests; + bare package import checks alone cannot prove real extraction isolation. + +## 根拠 + +- Tests: `tests/core/test_cells_reader_parity.py` (frozen previous-reader output, + actual BIFF and date-mode comparisons), + `tests/core/test_openpyxl_extraction_session.py` (loader counts, cleanup, + concurrency, overrides and BIFF routing), + `tests/core/test_backend_import_isolation.py` (real subprocess extraction), + `tests/cli/test_cli_lazy_imports.py`, and `tests/benchmark/test_performance.py`. +- Code: `src/exstruct/core/cells.py`, `src/exstruct/core/workbook.py`, + `src/exstruct/core/pipeline.py`, `src/exstruct/core/openpyxl_session.py`, + `src/exstruct/core/cell_types.py`, and `src/exstruct/core/backends/`. +- Related specs: `dev-docs/specs/excel-extraction.md`, + `dev-docs/specs/extraction-performance.md`, and `docs/api.md`. +- Related decisions: ADR-0002 fallback policy, ADR-0003 serialization policy, + and ADR-0010 light-mode responsibility boundary. + +## Supersedes + +- None + +## Superseded by + +- None diff --git a/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md b/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md new file mode 100644 index 00000000..a0b83cf7 --- /dev/null +++ b/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md @@ -0,0 +1,68 @@ +# ADR-0012: Optional SciPy Border Clustering + +## 状態 + +`proposed` + +## 背景 + +Issue #148 evaluates whether border clustering justifies a mandatory SciPy +dependency. NumPy is used beyond clustering and remains required. The existing +Python BFS already provides a fallback; removing that implementation or changing +backend names would expand compatibility scope unnecessarily. + +The coordinator compared SciPy, existing Python BFS and an evaluation-only sparse +set BFS on four deterministic workloads, with one warmup and seven timed repeats. +All bounding boxes, their order and full light extraction JSON matched exactly. +After reviewing these measurements, the user chose optional SciPy with the +existing Python fallback on 2026-10-03. + +## 決定 + +- Remove SciPy from mandatory runtime dependencies; offer `fast = ["scipy>=1.16.3"]` + and include SciPy in `all`. Keep NumPy in core. +- Retain the existing `EXSTRUCT_BORDER_CLUSTER_BACKEND` values and selection: + `auto` attempts SciPy, `python` uses the existing BFS, and the legacy `numpy` + name attempts SciPy-backed labeling. Import or accelerated execution failure + still falls back to Python. Do not add a new alias or change defaults. +- Keep sparse-set BFS as an internal evaluation candidate. Do not promote it to + production selection based on these results. + +## 影響 + +- Base installation becomes smaller and works without SciPy. Users who want the + previous accelerated installation can choose `exstruct[fast]` or `[all]`. +- Dense clustering remains much faster with SciPy: the measured clustering + medians were 0.217 ms (SciPy), 19.959 ms (Python), and 26.997 ms (candidate). + Full extraction plus JSON was 529 ms, 590 ms, and 524 ms respectively; + worksheet loading dominates much of this workload, and these sequential + synthetic timings must not be treated as universal speed guarantees. +- Many disconnected components favored Python: clustering medians were 53.113 ms + (SciPy) and 4.877 ms (Python), with extraction totals of 374 ms and 299 ms. + Large sparse candidate clustering improved independently, but whole extraction + did not establish a consistent candidate benefit. +- Keeping the legacy `numpy` name preserves configuration compatibility even + though it describes an implementation that also uses SciPy. Public docs must + state this clearly. Base installation still includes NumPy, xlrd and xlwings; + optional SciPy does not imply that every backend dependency is optional. + +## 根拠 + +- Tests: `tests/core/test_border_clustering_issue148.py` covers exact ordering, + blocked SciPy imports, accelerated failures and live overrides. +- Code: `src/exstruct/core/cells.py`, `pyproject.toml`, `uv.lock`, + `benchmark/issue148_clustering.py`, and `benchmark/issue148_scipy_free.py`. +- Related specs: `docs/api.md`, `dev-docs/specs/excel-extraction.md`, + `dev-docs/specs/extraction-performance.md`, and `benchmark/issue148-results.md`. +- Measurements: `benchmark/baselines/issue148-2026-10-03-windows.json` records + raw timings, input hashes, output hashes, dependency versions and source state. +- Related decisions: ADR-0010 retains the light OOXML baseline; ADR-0011 treats + direct cell reading and extraction-scoped resources separately. + +## Supersedes + +- None + +## Superseded by + +- None diff --git a/dev-docs/adr/README.md b/dev-docs/adr/README.md index bd49d030..8f8f8666 100644 --- a/dev-docs/adr/README.md +++ b/dev-docs/adr/README.md @@ -45,3 +45,5 @@ ADRs record what was decided, under which constraints, and which trade-offs were | `ADR-0008` | Extraction CLI Runtime Capability Validation | `accepted` | `cli` | | `ADR-0009` | Single CLI Skill for Agent Workflows | `proposed` | `agents` | | `ADR-0010` | Light Mode as the Pure-Python Rich OOXML Baseline | `accepted` | `extraction` | +| `ADR-0011` | Extraction-Scoped Workbook Reuse and Direct Cell Reading | `proposed` | `extraction` | +| `ADR-0012` | Optional SciPy Border Clustering | `proposed` | `performance` | diff --git a/dev-docs/adr/decision-map.md b/dev-docs/adr/decision-map.md index f97c6ee3..b85cb0ca 100644 --- a/dev-docs/adr/decision-map.md +++ b/dev-docs/adr/decision-map.md @@ -8,6 +8,8 @@ This document is a human-readable map for navigating ADRs by domain. - `ADR-0002` Rich Backend Fallback Policy (`accepted`) - `ADR-0008` Extraction CLI Runtime Capability Validation (`accepted`) - `ADR-0010` Light Mode as the Pure-Python Rich OOXML Baseline (`accepted`) +- `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) +- `ADR-0012` Optional SciPy Border Clustering (`proposed`) ## mode @@ -19,6 +21,8 @@ This document is a human-readable map for navigating ADRs by domain. - `ADR-0002` Rich Backend Fallback Policy (`accepted`) - `ADR-0004` Patch Backend Selection Policy (`accepted`) - `ADR-0010` Light Mode as the Pure-Python Rich OOXML Baseline (`accepted`) +- `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) +- `ADR-0012` Optional SciPy Border Clustering (`proposed`) ## fallback @@ -39,6 +43,13 @@ This document is a human-readable map for navigating ADRs by domain. - `ADR-0007` Editing CLI as Public Operational Interface (`accepted`) - `ADR-0008` Extraction CLI Runtime Capability Validation (`accepted`) - `ADR-0010` Light Mode as the Pure-Python Rich OOXML Baseline (`accepted`) +- `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) +- `ADR-0012` Optional SciPy Border Clustering (`proposed`) + +## performance + +- `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) +- `ADR-0012` Optional SciPy Border Clustering (`proposed`) ## mcp diff --git a/dev-docs/adr/index.yaml b/dev-docs/adr/index.yaml index dc629f9d..6917efcc 100644 --- a/dev-docs/adr/index.yaml +++ b/dev-docs/adr/index.yaml @@ -156,3 +156,36 @@ adrs: - docs/api.md - docs/cli.md - docs/mcp.md + - id: ADR-0011 + title: Extraction-Scoped Workbook Reuse and Direct Cell Reading + status: proposed + path: dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md + primary_domain: extraction + domains: + - extraction + - performance + - backend + - compatibility + supersedes: [] + superseded_by: [] + related_specs: + - dev-docs/specs/excel-extraction.md + - dev-docs/specs/extraction-performance.md + - docs/api.md + - id: ADR-0012 + title: Optional SciPy Border Clustering + status: proposed + path: dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md + primary_domain: performance + domains: + - performance + - extraction + - backend + - compatibility + supersedes: [] + superseded_by: [] + related_specs: + - docs/api.md + - dev-docs/specs/excel-extraction.md + - dev-docs/specs/extraction-performance.md + - benchmark/issue148-results.md diff --git a/dev-docs/adr/reviews/ADR-0011-0012-2026-10-03.md b/dev-docs/adr/reviews/ADR-0011-0012-2026-10-03.md new file mode 100644 index 00000000..d3df1ec7 --- /dev/null +++ b/dev-docs/adr/reviews/ADR-0011-0012-2026-10-03.md @@ -0,0 +1,61 @@ +# ADR-0011 / ADR-0012 implementation and design review evidence + +## Scope and structural prerequisite + +Independent Orca Codex review compared integrated `10f53cf` with base `5bcb0e4` +in two read-only passes. It reviewed direct cell/BIFF semantics, sparse storage, +session ownership/fallback/concurrency tests, lazy imports and rich OOXML, +optional SciPy metadata/CI, ADR-0011/0012 and related ADR-0002/0003/0010, public +`docs/api.md`, and the linked extraction specs/src/tests. + +ADR lint found no unresolved high/medium structural findings: both drafts are +`proposed`, have required sections, concrete Tests/Code/Related specs, balanced +consequences and consistent supersede sections. Design review followed lint. + +## Initial findings and resolution + +- Medium / evidence-risk / policy-drift: local backend imports bypassed explicit + legacy `pipeline.ComBackend`, `ComRichBackend` and `LibreOfficeRichBackend` + overrides. This contradicted ADR-0011's live-override compatibility claim. + The initial reviewer verdict was `revise`; no new policy was needed. +- Low / index consistency: ADR-0012 listed `benchmark/issue148-results.md` as + Related specs but its derived YAML entry omitted that path. + +Commit `4952ed9` fixes both. `_backend_type` honors explicitly bound pipeline +aliases and otherwise resolves the concrete backend lazily. The regression +subprocess checks all four affected COM steps and both rich-backend selections +without importing concrete COM/LibreOffice modules. ADR-0012's index reference +now matches the source. + +## Independent focused verification + +A separate read-only verification task targeted exact commit +`4952ed9ee21a622d45dceba3d5d3ffaf70cb4219`, limited to closing those findings. +It confirmed explicit overrides, the default lazy/live resolution of all three +aliases, and a fresh-process real light extraction that returned 33 shapes +without loading the tested COM/render roots. It confirmed the source/index +benchmark reference and current lint prerequisites. + +- adr-reviewer verdict for the resolved claims: **ready**. +- adr-reconciler findings for those claims: **none**. +- Open questions in the focused scope: **none**. +- ADR statuses remain `proposed`; a ready review does not change their status. + +## Validation boundaries + +The independent review did not run full suites, actual COM/LibreOffice, timing, +wheel installs or remote CI. The coordinator separately verified the final +non-COM suite (1017 passed, coverage 83.09%), actual COM (8 passed), forced +LibreOffice (1 passed), PDF/PNG rendering (3 passed), installed base/fast wheels, +and exact output parity for the twelve measured input/mode combinations. +See `benchmark/issues145-148-results.md` for measurements and their limits. +Remote CI is distinct from this local evidence. This focused confirmation is +not a new broad review of unrelated code or a statistical performance approval. + +The later Codacy follow-up replaces generic dynamic imports and globals-index +lookup with fixed imports and module attributes, and replaces session assertions +with normal guards. It preserves the same existing policy; no new ADR is needed. +Its non-COM suite passed 1017 tests with 83.04% coverage, including the override, +session and real light import regressions. The independent verdict above remains +explicitly scoped to `4952ed9`; later static-analysis changes are coordinator +validation rather than a claim of another independent review. diff --git a/dev-docs/architecture/pipeline.md b/dev-docs/architecture/pipeline.md index 116c4eb2..2703e682 100644 --- a/dev-docs/architecture/pipeline.md +++ b/dev-docs/architecture/pipeline.md @@ -58,9 +58,19 @@ Pipeline is the **orchestrator**. - Determines the extraction order - Selects backends - Controls fallback paths -- Manages intermediate artifacts +- Manages intermediate artifacts +- Owns one extraction-scoped openpyxl session across pre-analysis, COM/fallback + processing and final model construction. Backends consume that session rather + than opening the same workbook for each feature or worksheet. -Pipeline is designed to **never read Excel content directly**. +Pipeline is designed to **never read Excel content directly**. + +`OpenpyxlExtractionSession` lazily owns regular workbook variants through an +`ExitStack`: cached values (`data_only=True`) are shared by cells, hyperlinks, +print areas, colors, merged cells and table detection; formula text +(`data_only=False`) is opened only when requested. Path-based standalone helpers +remain wrappers with their own bounded lifetime. The session is not a global +cache and must not outlive one extraction. --- diff --git a/dev-docs/specs/excel-extraction.md b/dev-docs/specs/excel-extraction.md index 423e8d21..18c677f5 100644 --- a/dev-docs/specs/excel-extraction.md +++ b/dev-docs/specs/excel-extraction.md @@ -12,6 +12,24 @@ This document summarizes the current specification for Excel extraction processi 6. When COM succeeds, colors_map is overwritten with COM results 7. When the rich backend fails, cells+table_candidates are preserved, and pre-com artifacts (print_areas / formulas_map / colors_map / merged_cells) are also retained according to their flags +## Workbook Resource Lifetime + +- One extraction invocation owns an `OpenpyxlExtractionSession`. Compatible + OOXML stages share one regular `data_only=True`, `read_only=False` workbook, + including hyperlink and per-sheet table detection, until final modeling ends. +- An openpyxl formula map opens a second regular `data_only=False` workbook + lazily. Disabled formula extraction and COM-only `.xls` formula extraction + must not open that variant. +- Without helper overrides, an OOXML extraction loads one openpyxl workbook, or + two when openpyxl formula extraction is enabled, independent of sheet count. + OOXML drawing ZIP reads are separate and are not included in that loader count. +- Standalone path-based helper calls own and close their own workbooks. An + extraction does not cache workbooks across invocations or share them with + another concurrent extraction. +- Success, fallback and exceptions release all successfully opened variants. + Existing helper overrides remain observable and may intentionally replace + the optimized session path. + ## Coordinate System - Rows are 1-based @@ -24,16 +42,54 @@ This document summarizes the current specification for Excel extraction processi - standard: Existing behavior (text-bearing shapes, charts if needed) - verbose: All shapes + sizes, charts with sizes +## Import Boundaries + +- Bare package/engine imports and CLI help keep their existing lazy contracts. + Real `.xlsx/.xlsm` light extraction loads neither xlwings nor concrete COM + backends, COM shape/chart modules, ExStruct rendering or PDFium. +- Shared cell/map/merged-range types do not import concrete extraction backends. + Compatibility exports and override call sites resolve the live implementation + only when requested or executed. +- Openpyxl and NumPy remain available when extraction needs them. Openpyxl may + import optional Pillow itself when installed; light extraction must also work + without Pillow. This transitive import does not activate ExStruct rendering. +- BIFF xlrd is imported only for `.xls` cell reading. SciPy labeling is imported + only when accelerated clustering is attempted; explicit Python clustering + does not import SciPy. + ## Cell Extraction -- Load with pandas `read_excel(header=None, dtype=str)` +- Read `.xlsx` / `.xlsm` cached cell values directly with openpyxl and `.xls` + cached values directly with xlrd; no production pandas reader is used. - Ignore blank cells - Normalize row data into `CellRow` +- Preserve existing numeric coercion (including zero-prefixed numeric strings), boolean/date/time text, + row gaps, sheet ordering and the former reader's default missing-string tokens. + Excel error cells are omitted as before. Missing tokens are matched before + trimming, so padded tokens remain ordinary text. +- Hyperlinks use their external `target` and zero-based numeric-string column + keys. Links on filtered cells are retained when their row contains another + emitted value; a row containing only filtered values is not created for links. +- Formula text extraction remains separate from cell extraction. Cell values + use stored formula results and do not calculate formulas. +- `.xls` direct reading adds no OOXML rich artifacts or COM-independent formula, + hyperlink, color, merged-cell or table extraction guarantees. ## Table Extraction - Merge openpyxl table definitions + border clusters - Preserve table_candidates even when COM is unavailable +- Path-based detection delegates to worksheet-based detection. Nested border + scanning and multi-sheet detection must not reopen the workbook when the + caller supplies a worksheet from the extraction session. +- COM table detection reuses a supplied session only when its resolved file + path matches the COM workbook path. A different workbook uses standalone + path-based detection, even when both workbooks contain the same sheet name. +- NumPy remains required. SciPy is optional through `exstruct[fast]` or `[all]`. + `EXSTRUCT_BORDER_CLUSTER_BACKEND=auto` attempts SciPy-backed labeling, while + `python` forces the existing Python BFS. The legacy `numpy` value also attempts + SciPy; import or execution failure falls back to Python. Unknown values retain + the existing `auto` behavior. Sparse-set BFS remains an evaluation candidate. ## Shapes / Arrows / SmartArt Extraction diff --git a/dev-docs/specs/extraction-performance.md b/dev-docs/specs/extraction-performance.md index ce413969..5e5cd64e 100644 --- a/dev-docs/specs/extraction-performance.md +++ b/dev-docs/specs/extraction-performance.md @@ -134,3 +134,10 @@ test measurement contracts without strict shared-runner wall-clock thresholds. The committed baseline's `environment.executable` is replaced with `` to omit the identifying local path; local runner output still records its actual interpreter path. Timing samples and other reproducibility metadata are retained. + +The integrated #145-#148 comparison and exact output hashes are recorded in +`benchmark/issues145-148-results.md` and its linked raw baselines. It separates +light improvements from variable Excel COM observations and retains supplemental +paired rechecks. `benchmark/issue148-results.md` records the separate accelerator +comparison and ADR-0012 dependency decision; the candidate sparse BFS remains +evaluation-only. diff --git a/docs/api.md b/docs/api.md index 07a37062..bafa6aa5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -147,7 +147,13 @@ the MCP server, which wraps the same core and adds host policy. ## Dependencies -- Core extraction: pandas, openpyxl (installed with the package). +- Core cell extraction: openpyxl for `.xlsx` / `.xlsm`, xlrd for `.xls` + (installed with the package). Production extraction does not require pandas. +- Border clustering: NumPy is required; SciPy is optional via `exstruct[fast]` + (also included in `exstruct[all]`). `EXSTRUCT_BORDER_CLUSTER_BACKEND=auto` + prefers SciPy when available and falls back to the existing Python BFS on + import or execution failure. `python` forces BFS; the legacy `numpy` name + selects SciPy-backed labeling with the same fallback. No `scipy` alias is added. - YAML export: `pyyaml` (lazy import; missing module raises `MissingDependencyError`). - TOON export: `python-toon` (lazy import; missing module raises `MissingDependencyError`). - Auto page-break extraction/export: **Excel + COM** required. `mode="libreoffice"` rejects auto page-break requests with `ConfigError`. diff --git a/pyproject.toml b/pyproject.toml index b3cce549..be1a360f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,9 +13,8 @@ dependencies = [ "defusedxml>=0.7.1", "numpy>=2.3.5", "openpyxl>=3.1.5", - "pandas>=2.3.3", "pydantic>=2.12.5", - "scipy>=1.16.3", + "xlrd>=2.0.2", "xlwings>=0.33.16", ] @@ -28,7 +27,8 @@ dev = [ "codecov-cli>=11.2.6", "mkdocs-material>=9.7.0", "mkdocstrings-python>=2.0.1", - "mypy>=1.19.0", + "mypy>=1.19.0", + "pandas>=2.3.3", "pre-commit>=4.5.0", "pytest>=9.0.1", "pytest-cov>=7.0.0", @@ -37,8 +37,10 @@ dev = [ "taskipy>=1.14.1", ] -[project.optional-dependencies] -all = [ +[project.optional-dependencies] +fast = ["scipy>=1.16.3"] +all = [ + "scipy>=1.16.3", "pyyaml>=6.0.3", "python-toon>=0.1.3", "pypdfium2>=5.1.0", diff --git a/src/exstruct/core/_lazy.py b/src/exstruct/core/_lazy.py new file mode 100644 index 00000000..fa54cdcd --- /dev/null +++ b/src/exstruct/core/_lazy.py @@ -0,0 +1,27 @@ +"""Deferred module access for compatibility and runtime annotation resolution.""" + +from typing import Any, Literal + + +class LazyModule: + """Load xlwings only when one of its attributes is actually requested.""" + + def __init__(self, name: Literal["xlwings"]) -> None: + if name != "xlwings": + raise ValueError("Only the xlwings compatibility module is supported") + + def __getattr__(self, name: str) -> Any: # noqa: ANN401 - dependency attributes vary + import xlwings + + return getattr(xlwings, name) + + def __setattr__(self, name: str, value: object) -> None: + """Keep legacy monkeypatches visible to other users of the real module.""" + import xlwings + + setattr(xlwings, name, value) + + def __delattr__(self, name: str) -> None: + import xlwings + + delattr(xlwings, name) diff --git a/src/exstruct/core/backends/__init__.py b/src/exstruct/core/backends/__init__.py index 9fc3da3e..345f77be 100644 --- a/src/exstruct/core/backends/__init__.py +++ b/src/exstruct/core/backends/__init__.py @@ -2,11 +2,9 @@ from __future__ import annotations -from .base import Backend -from .com_backend import ComBackend, ComRichBackend -from .libreoffice_backend import LibreOfficeRichBackend -from .ooxml_backend import OoxmlRichBackend -from .openpyxl_backend import OpenpyxlBackend +from typing import Any + +from .base import Backend as Backend __all__ = [ "Backend", @@ -16,3 +14,24 @@ "OoxmlRichBackend", "OpenpyxlBackend", ] + + +def __getattr__(name: str) -> Any: # noqa: ANN401 - heterogeneous backend exports + """Resolve the requested backend export without importing other backends.""" + if name in {"ComBackend", "ComRichBackend"}: + from . import com_backend + + return getattr(com_backend, name) + if name == "LibreOfficeRichBackend": + from . import libreoffice_backend + + return libreoffice_backend.LibreOfficeRichBackend + if name == "OoxmlRichBackend": + from . import ooxml_backend + + return ooxml_backend.OoxmlRichBackend + if name == "OpenpyxlBackend": + from . import openpyxl_backend + + return openpyxl_backend.OpenpyxlBackend + raise AttributeError(name) diff --git a/src/exstruct/core/backends/base.py b/src/exstruct/core/backends/base.py index 85f395f0..5fb5c43e 100644 --- a/src/exstruct/core/backends/base.py +++ b/src/exstruct/core/backends/base.py @@ -6,7 +6,7 @@ from typing import Literal, Protocol from ...models import Arrow, CellRow, Chart, PrintArea, Shape, SmartArt -from ..cells import MergedCellRange, WorkbookColorsMap, WorkbookFormulasMap +from ..cell_types import MergedCellRange, WorkbookColorsMap, WorkbookFormulasMap CellData = dict[str, list[CellRow]] PrintAreaData = dict[str, list[PrintArea]] diff --git a/src/exstruct/core/backends/libreoffice_backend.py b/src/exstruct/core/backends/libreoffice_backend.py index 8da0fef5..069c7eda 100644 --- a/src/exstruct/core/backends/libreoffice_backend.py +++ b/src/exstruct/core/backends/libreoffice_backend.py @@ -5,7 +5,6 @@ from collections.abc import Callable, Sequence from contextlib import AbstractContextManager import logging -import math from pathlib import Path from typing import Protocol, TypeGuard @@ -17,8 +16,22 @@ LibreOfficeWorkbookHandle, ) from ..ooxml_drawing import OoxmlConnectorInfo, OoxmlShapeInfo, read_sheet_drawings -from ..shapes import angle_to_compass, compute_line_angle_deg from .base import ChartData, RichBackend, ShapeData +from .ooxml_shapes import ( # noqa: F401 - legacy compatibility re-exports + _build_shapes_from_ooxml as _build_shapes_from_ooxml, + _classify_connector_resolution as _classify_connector_resolution, # noqa: F401 - legacy re-export + _connector_endpoints as _connector_endpoints, + _direction_from_shape_boxes as _direction_from_shape_boxes, + _distance_to_box as _distance_to_box, + _nearest_shape_id as _nearest_shape_id, + _resolve_connector as _resolve_connector, + _resolve_direct_connector_ids as _resolve_direct_connector_ids, + _resolve_direction as _resolve_direction, + _rotate_connector_delta as _rotate_connector_delta, + _shape_box_center as _shape_box_center, + _ShapeBox as _ShapeBox, + _to_shape_box as _to_shape_box, +) logger = logging.getLogger(__name__) @@ -258,90 +271,6 @@ def _extract_draw_page_shapes_with_optional_workbook_lifecycle( session.close_workbook(workbook) -def _build_shapes_from_ooxml( - shapes: Sequence[OoxmlShapeInfo], - connectors: Sequence[OoxmlConnectorInfo], - *, - provenance: str = "libreoffice_uno", -) -> list[Shape | Arrow | SmartArt]: - """Build emitted shape models directly from OOXML drawing metadata. - - Args: - shapes: Parsed OOXML shape candidates. - connectors: Parsed OOXML connector candidates. - - Returns: - Emitted shape and arrow models derived from the OOXML drawing anchors. - """ - - emitted: list[Shape | Arrow | SmartArt] = [] - drawing_to_shape_id: dict[int, int] = {} - shape_boxes: dict[int, _ShapeBox] = {} - next_shape_id = 0 - for shape_info in shapes: - next_shape_id += 1 - shape_id = next_shape_id - drawing_to_shape_id[shape_info.ref.drawing_id] = shape_id - box = _to_shape_box( - shape_id=shape_id, - left=shape_info.ref.left, - top=shape_info.ref.top, - width=shape_info.ref.width, - height=shape_info.ref.height, - ) - if box is not None: - shape_boxes[shape_id] = box - emitted.append( - Shape( - id=shape_id, - text=shape_info.text, - l=shape_info.ref.left or 0, - t=shape_info.ref.top or 0, - w=shape_info.ref.width, - h=shape_info.ref.height, - rotation=shape_info.rotation, - type=shape_info.shape_type, - provenance=provenance, - approximation_level="partial", - confidence=0.75, - ) - ) - for connector_info in connectors: - begin_id, end_id, approximation_level, confidence = _resolve_connector( - connector_info, - uno_connector=None, - drawing_to_shape_id=drawing_to_shape_id, - shape_name_to_id={}, - shape_boxes=shape_boxes, - ) - emitted.append( - Arrow( - id=None, - text=connector_info.text, - l=connector_info.ref.left or 0, - t=connector_info.ref.top or 0, - w=connector_info.ref.width, - h=connector_info.ref.height, - rotation=connector_info.rotation, - begin_arrow_style=connector_info.begin_arrow_style, - end_arrow_style=connector_info.end_arrow_style, - begin_id=begin_id, - end_id=end_id, - direction=_resolve_direction( - connector_info=connector_info, - uno_connector=None, - begin_id=begin_id, - end_id=end_id, - shape_boxes=shape_boxes, - ), - provenance=provenance, - approximation_level=approximation_level, - confidence=confidence, - ) - ) - return emitted - - def _build_shapes_from_draw_page( snapshots: Sequence[LibreOfficeDrawPageShape], *, @@ -523,207 +452,6 @@ def _log_unmatched_ooxml_candidates( ) -def _resolve_connector( - connector_info: OoxmlConnectorInfo | None, - *, - uno_connector: LibreOfficeDrawPageShape | None, - drawing_to_shape_id: dict[int, int], - shape_name_to_id: dict[str, int], - shape_boxes: dict[int, _ShapeBox], -) -> tuple[int | None, int | None, str, float]: - """Resolve connector endpoints using OOXML refs, UNO refs, or geometry heuristics. - - Args: - connector_info: OOXML connector metadata when available. - uno_connector: LibreOffice draw-page connector snapshot when available. - drawing_to_shape_id: Mapping from OOXML drawing ids to emitted shape ids. - shape_name_to_id: Mapping from UNO shape names to emitted shape ids. - shape_boxes: Bounding boxes used for heuristic endpoint matching. - - Returns: - Tuple of begin id, end id, approximation label, and confidence score. - """ - - begin_id, end_id_resolved, used_ooxml_direct, used_uno_direct = ( - _resolve_direct_connector_ids( - connector_info=connector_info, - uno_connector=uno_connector, - drawing_to_shape_id=drawing_to_shape_id, - shape_name_to_id=shape_name_to_id, - ) - ) - - if begin_id is not None and end_id_resolved is not None: - return _classify_connector_resolution( - begin_id=begin_id, - end_id=end_id_resolved, - used_ooxml_direct=used_ooxml_direct, - used_uno_direct=used_uno_direct, - used_heuristic=False, - ) - - start_point, end_point = _connector_endpoints( - connector_info=connector_info, - uno_connector=uno_connector, - ) - if begin_id is None: - begin_id = _nearest_shape_id(start_point, shape_boxes) - if end_id_resolved is None: - end_id_resolved = _nearest_shape_id(end_point, shape_boxes) - return _classify_connector_resolution( - begin_id=begin_id, - end_id=end_id_resolved, - used_ooxml_direct=used_ooxml_direct, - used_uno_direct=used_uno_direct, - used_heuristic=True, - ) - - -def _resolve_direction( - *, - connector_info: OoxmlConnectorInfo | None, - uno_connector: LibreOfficeDrawPageShape | None, - begin_id: int | None = None, - end_id: int | None = None, - shape_boxes: dict[int, _ShapeBox] | None = None, -) -> str | None: - """Infer connector direction from OOXML deltas or resolved endpoint geometry.""" - - if connector_info is None: - return _direction_from_shape_boxes( - begin_id=begin_id, - end_id=end_id, - shape_boxes=shape_boxes, - ) - dx = connector_info.direction_dx - dy = connector_info.direction_dy - if dx is None or dy is None: - return _direction_from_shape_boxes( - begin_id=begin_id, - end_id=end_id, - shape_boxes=shape_boxes, - ) - if dx == 0 and dy == 0: - return _direction_from_shape_boxes( - begin_id=begin_id, - end_id=end_id, - shape_boxes=shape_boxes, - ) - rotated_dx, rotated_dy = _rotate_connector_delta( - float(dx), - float(dy), - connector_info.rotation, - ) - angle = compute_line_angle_deg(rotated_dx, rotated_dy) - return angle_to_compass(angle) - - -def _connector_endpoints( - *, - connector_info: OoxmlConnectorInfo | None, - uno_connector: LibreOfficeDrawPageShape | None, -) -> tuple[tuple[float, float] | None, tuple[float, float] | None]: - """Return connector endpoints for heuristic endpoint matching.""" - - if connector_info is not None: - left = connector_info.ref.left - top = connector_info.ref.top - dx = connector_info.direction_dx - dy = connector_info.direction_dy - if ( - left is not None - and top is not None - and dx is not None - and dy is not None - and (dx != 0 or dy != 0) - ): - rotated_dx, rotated_dy = _rotate_connector_delta( - float(dx), - float(dy), - connector_info.rotation, - ) - start = (float(left), float(top)) - end = (float(left) + rotated_dx, float(top) + rotated_dy) - return (start, end) - - if uno_connector is None: - return (None, None) - left = uno_connector.left - top = uno_connector.top - width = uno_connector.width - height = uno_connector.height - if left is None or top is None or width is None or height is None: - return (None, None) - start = (float(left), float(top)) - end = (float(left + width), float(top + height)) - return (start, end) - - -def _nearest_shape_id( - point: tuple[float, float] | None, shape_boxes: dict[int, _ShapeBox] -) -> int | None: - """Return the closest emitted shape id to a point.""" - - if point is None or not shape_boxes: - return None - x, y = point - best_shape_id: int | None = None - best_distance: float | None = None - for shape_id, box in shape_boxes.items(): - distance = _distance_to_box(x, y, box) - if best_distance is None or distance < best_distance: - best_distance = distance - best_shape_id = shape_id - return best_shape_id - - -def _distance_to_box(x: float, y: float, box: _ShapeBox) -> float: - """Compute the Euclidean distance from a point to a shape box.""" - - dx = max(box.left - x, 0.0, x - box.right) - dy = max(box.top - y, 0.0, y - box.bottom) - return math.hypot(dx, dy) - - -def _rotate_connector_delta( - dx: float, - dy: float, - rotation_deg: float | None, -) -> tuple[float, float]: - """Rotate an OOXML connector delta into sheet coordinates when needed.""" - - if rotation_deg is None: - return (dx, dy) - if math.isclose(rotation_deg % 360.0, 0.0, abs_tol=1e-9): - return (dx, dy) - length = math.hypot(dx, dy) - if length == 0.0: - return (dx, dy) - angle_rad = math.radians(compute_line_angle_deg(dx, dy) + rotation_deg) - return (length * math.cos(angle_rad), length * math.sin(angle_rad)) - - -def _to_shape_box( - *, - shape_id: int, - left: int | None, - top: int | None, - width: int | None, - height: int | None, -) -> _ShapeBox | None: - """Build a shape box when complete rectangle geometry is available.""" - - if left is None or top is None or width is None or height is None: - return None - return _ShapeBox( - shape_id=shape_id, - left=float(left), - top=float(top), - right=float(left + width), - bottom=float(top + height), - ) - - def _match_shape_infos( snapshots: Sequence[LibreOfficeDrawPageShape], candidates: Sequence[OoxmlShapeInfo], @@ -797,86 +525,6 @@ def _shape_type_from_uno(shape_type: str | None) -> str | None: return shape_type.rsplit(".", 1)[-1] -def _direction_from_shape_boxes( - *, - begin_id: int | None, - end_id: int | None, - shape_boxes: dict[int, _ShapeBox] | None, -) -> str | None: - """Infer a connector direction from resolved endpoint shape centers.""" - - if begin_id is None or end_id is None or shape_boxes is None: - return None - begin_box = shape_boxes.get(begin_id) - end_box = shape_boxes.get(end_id) - if begin_box is None or end_box is None: - return None - begin_center = _shape_box_center(begin_box) - end_center = _shape_box_center(end_box) - dx = end_center[0] - begin_center[0] - dy = end_center[1] - begin_center[1] - if dx == 0 and dy == 0: - return None - angle = compute_line_angle_deg(dx, dy) - return angle_to_compass(angle) - - -def _shape_box_center(box: _ShapeBox) -> tuple[float, float]: - """Return the center point of a matched emitted shape box.""" - - return ((box.left + box.right) / 2.0, (box.top + box.bottom) / 2.0) - - -def _resolve_direct_connector_ids( - *, - connector_info: OoxmlConnectorInfo | None, - uno_connector: LibreOfficeDrawPageShape | None, - drawing_to_shape_id: dict[int, int], - shape_name_to_id: dict[str, int], -) -> tuple[int | None, int | None, bool, bool]: - """Resolve direct connector endpoints from OOXML refs and UNO shape refs.""" - - begin_id: int | None = None - end_id: int | None = None - used_ooxml_direct = False - used_uno_direct = False - if connector_info is not None: - start_id = connector_info.connection.start_drawing_id - target_id = connector_info.connection.end_drawing_id - begin_id = drawing_to_shape_id.get(start_id) if start_id is not None else None - end_id = drawing_to_shape_id.get(target_id) if target_id is not None else None - used_ooxml_direct = begin_id is not None or end_id is not None - if uno_connector is not None: - if begin_id is None and uno_connector.start_shape_name is not None: - begin_id = shape_name_to_id.get(uno_connector.start_shape_name) - used_uno_direct = used_uno_direct or begin_id is not None - if end_id is None and uno_connector.end_shape_name is not None: - end_id = shape_name_to_id.get(uno_connector.end_shape_name) - used_uno_direct = used_uno_direct or end_id is not None - return (begin_id, end_id, used_ooxml_direct, used_uno_direct) - - -def _classify_connector_resolution( - *, - begin_id: int | None, - end_id: int | None, - used_ooxml_direct: bool, - used_uno_direct: bool, - used_heuristic: bool, -) -> tuple[int | None, int | None, str, float]: - """Classify connector provenance once endpoint resolution is complete.""" - - if used_heuristic: - return (begin_id, end_id, "heuristic", 0.6) - if used_ooxml_direct and used_uno_direct: - return (begin_id, end_id, "partial", 0.9) - if used_ooxml_direct: - return (begin_id, end_id, "direct", 1.0) - if used_uno_direct: - return (begin_id, end_id, "direct", 0.9) - return (begin_id, end_id, "heuristic", 0.6) - - def _match_chart_geometries( charts: Sequence[object], candidates: Sequence[LibreOfficeChartGeometry], @@ -941,18 +589,3 @@ def _first_optional_float(*values: float | None) -> float | None: if value is not None: return value return None - - -class _ShapeBox: - """Axis-aligned bounding box for emitted shapes.""" - - def __init__( - self, *, shape_id: int, left: float, top: float, right: float, bottom: float - ) -> None: - """Store bounding-box coordinates for emitted shape matching.""" - - self.shape_id = shape_id - self.left = left - self.top = top - self.right = right - self.bottom = bottom diff --git a/src/exstruct/core/backends/ooxml_backend.py b/src/exstruct/core/backends/ooxml_backend.py index a38b78ed..15de21c0 100644 --- a/src/exstruct/core/backends/ooxml_backend.py +++ b/src/exstruct/core/backends/ooxml_backend.py @@ -11,7 +11,7 @@ from ...models import Chart from ..ooxml_drawing import SheetDrawingData, read_sheet_drawings from .base import ChartData, RichBackend, ShapeData -from .libreoffice_backend import _build_shapes_from_ooxml +from .ooxml_shapes import _build_shapes_from_ooxml logger = logging.getLogger(__name__) diff --git a/src/exstruct/core/backends/ooxml_shapes.py b/src/exstruct/core/backends/ooxml_shapes.py new file mode 100644 index 00000000..8aef91d6 --- /dev/null +++ b/src/exstruct/core/backends/ooxml_shapes.py @@ -0,0 +1,413 @@ +"""Shared OOXML shape construction and connector geometry; no COM imports.""" + +from __future__ import annotations + +from collections.abc import Sequence +import math +from typing import Protocol + +from ...models import Arrow, Shape, SmartArt +from ..drawing_geometry import angle_to_compass, compute_line_angle_deg +from ..ooxml_drawing import OoxmlConnectorInfo, OoxmlShapeInfo + + +class ConnectorSnapshot(Protocol): + """Geometry and direct references needed from optional enrichment snapshots.""" + + @property + def left(self) -> int | None: ... + + @property + def top(self) -> int | None: ... + + @property + def width(self) -> int | None: ... + + @property + def height(self) -> int | None: ... + + @property + def start_shape_name(self) -> str | None: ... + + @property + def end_shape_name(self) -> str | None: ... + + +def _build_shapes_from_ooxml( + shapes: Sequence[OoxmlShapeInfo], + connectors: Sequence[OoxmlConnectorInfo], + *, + provenance: str = "libreoffice_uno", +) -> list[Shape | Arrow | SmartArt]: + """Build emitted shape models directly from OOXML drawing metadata. + + Args: + shapes: Parsed OOXML shape candidates. + connectors: Parsed OOXML connector candidates. + + Returns: + Emitted shape and arrow models derived from the OOXML drawing anchors. + """ + + emitted: list[Shape | Arrow | SmartArt] = [] + drawing_to_shape_id: dict[int, int] = {} + shape_boxes: dict[int, _ShapeBox] = {} + next_shape_id = 0 + for shape_info in shapes: + next_shape_id += 1 + shape_id = next_shape_id + drawing_to_shape_id[shape_info.ref.drawing_id] = shape_id + box = _to_shape_box( + shape_id=shape_id, + left=shape_info.ref.left, + top=shape_info.ref.top, + width=shape_info.ref.width, + height=shape_info.ref.height, + ) + if box is not None: + shape_boxes[shape_id] = box + emitted.append( + Shape( + id=shape_id, + text=shape_info.text, + l=shape_info.ref.left or 0, + t=shape_info.ref.top or 0, + w=shape_info.ref.width, + h=shape_info.ref.height, + rotation=shape_info.rotation, + type=shape_info.shape_type, + provenance=provenance, + approximation_level="partial", + confidence=0.75, + ) + ) + for connector_info in connectors: + begin_id, end_id, approximation_level, confidence = _resolve_connector( + connector_info, + uno_connector=None, + drawing_to_shape_id=drawing_to_shape_id, + shape_name_to_id={}, + shape_boxes=shape_boxes, + ) + emitted.append( + Arrow( + id=None, + text=connector_info.text, + l=connector_info.ref.left or 0, + t=connector_info.ref.top or 0, + w=connector_info.ref.width, + h=connector_info.ref.height, + rotation=connector_info.rotation, + begin_arrow_style=connector_info.begin_arrow_style, + end_arrow_style=connector_info.end_arrow_style, + begin_id=begin_id, + end_id=end_id, + direction=_resolve_direction( + connector_info=connector_info, + uno_connector=None, + begin_id=begin_id, + end_id=end_id, + shape_boxes=shape_boxes, + ), + provenance=provenance, + approximation_level=approximation_level, + confidence=confidence, + ) + ) + return emitted + + +def _resolve_connector( + connector_info: OoxmlConnectorInfo | None, + *, + uno_connector: ConnectorSnapshot | None, + drawing_to_shape_id: dict[int, int], + shape_name_to_id: dict[str, int], + shape_boxes: dict[int, _ShapeBox], +) -> tuple[int | None, int | None, str, float]: + """Resolve connector endpoints using OOXML refs, UNO refs, or geometry heuristics. + + Args: + connector_info: OOXML connector metadata when available. + uno_connector: LibreOffice draw-page connector snapshot when available. + drawing_to_shape_id: Mapping from OOXML drawing ids to emitted shape ids. + shape_name_to_id: Mapping from UNO shape names to emitted shape ids. + shape_boxes: Bounding boxes used for heuristic endpoint matching. + + Returns: + Tuple of begin id, end id, approximation label, and confidence score. + """ + + begin_id, end_id_resolved, used_ooxml_direct, used_uno_direct = ( + _resolve_direct_connector_ids( + connector_info=connector_info, + uno_connector=uno_connector, + drawing_to_shape_id=drawing_to_shape_id, + shape_name_to_id=shape_name_to_id, + ) + ) + + if begin_id is not None and end_id_resolved is not None: + return _classify_connector_resolution( + begin_id=begin_id, + end_id=end_id_resolved, + used_ooxml_direct=used_ooxml_direct, + used_uno_direct=used_uno_direct, + used_heuristic=False, + ) + + start_point, end_point = _connector_endpoints( + connector_info=connector_info, + uno_connector=uno_connector, + ) + if begin_id is None: + begin_id = _nearest_shape_id(start_point, shape_boxes) + if end_id_resolved is None: + end_id_resolved = _nearest_shape_id(end_point, shape_boxes) + return _classify_connector_resolution( + begin_id=begin_id, + end_id=end_id_resolved, + used_ooxml_direct=used_ooxml_direct, + used_uno_direct=used_uno_direct, + used_heuristic=True, + ) + + +def _resolve_direction( + *, + connector_info: OoxmlConnectorInfo | None, + uno_connector: ConnectorSnapshot | None, + begin_id: int | None = None, + end_id: int | None = None, + shape_boxes: dict[int, _ShapeBox] | None = None, +) -> str | None: + """Infer connector direction from OOXML deltas or resolved endpoint geometry.""" + + if connector_info is None: + return _direction_from_shape_boxes( + begin_id=begin_id, + end_id=end_id, + shape_boxes=shape_boxes, + ) + dx = connector_info.direction_dx + dy = connector_info.direction_dy + if dx is None or dy is None: + return _direction_from_shape_boxes( + begin_id=begin_id, + end_id=end_id, + shape_boxes=shape_boxes, + ) + if dx == 0 and dy == 0: + return _direction_from_shape_boxes( + begin_id=begin_id, + end_id=end_id, + shape_boxes=shape_boxes, + ) + rotated_dx, rotated_dy = _rotate_connector_delta( + float(dx), + float(dy), + connector_info.rotation, + ) + angle = compute_line_angle_deg(rotated_dx, rotated_dy) + return angle_to_compass(angle) + + +def _connector_endpoints( + *, + connector_info: OoxmlConnectorInfo | None, + uno_connector: ConnectorSnapshot | None, +) -> tuple[tuple[float, float] | None, tuple[float, float] | None]: + """Return connector endpoints for heuristic endpoint matching.""" + + if connector_info is not None: + left = connector_info.ref.left + top = connector_info.ref.top + dx = connector_info.direction_dx + dy = connector_info.direction_dy + if ( + left is not None + and top is not None + and dx is not None + and dy is not None + and (dx != 0 or dy != 0) + ): + rotated_dx, rotated_dy = _rotate_connector_delta( + float(dx), + float(dy), + connector_info.rotation, + ) + start = (float(left), float(top)) + end = (float(left) + rotated_dx, float(top) + rotated_dy) + return (start, end) + + if uno_connector is None: + return (None, None) + left = uno_connector.left + top = uno_connector.top + width = uno_connector.width + height = uno_connector.height + if left is None or top is None or width is None or height is None: + return (None, None) + start = (float(left), float(top)) + end = (float(left + width), float(top + height)) + return (start, end) + + +def _nearest_shape_id( + point: tuple[float, float] | None, shape_boxes: dict[int, _ShapeBox] +) -> int | None: + """Return the closest emitted shape id to a point.""" + + if point is None or not shape_boxes: + return None + x, y = point + best_shape_id: int | None = None + best_distance: float | None = None + for shape_id, box in shape_boxes.items(): + distance = _distance_to_box(x, y, box) + if best_distance is None or distance < best_distance: + best_distance = distance + best_shape_id = shape_id + return best_shape_id + + +def _distance_to_box(x: float, y: float, box: _ShapeBox) -> float: + """Compute the Euclidean distance from a point to a shape box.""" + + dx = max(box.left - x, 0.0, x - box.right) + dy = max(box.top - y, 0.0, y - box.bottom) + return math.hypot(dx, dy) + + +def _rotate_connector_delta( + dx: float, + dy: float, + rotation_deg: float | None, +) -> tuple[float, float]: + """Rotate an OOXML connector delta into sheet coordinates when needed.""" + + if rotation_deg is None: + return (dx, dy) + if math.isclose(rotation_deg % 360.0, 0.0, abs_tol=1e-9): + return (dx, dy) + length = math.hypot(dx, dy) + if length == 0.0: + return (dx, dy) + angle_rad = math.radians(compute_line_angle_deg(dx, dy) + rotation_deg) + return (length * math.cos(angle_rad), length * math.sin(angle_rad)) + + +def _to_shape_box( + *, + shape_id: int, + left: int | None, + top: int | None, + width: int | None, + height: int | None, +) -> _ShapeBox | None: + """Build a shape box when complete rectangle geometry is available.""" + + if left is None or top is None or width is None or height is None: + return None + return _ShapeBox( + shape_id=shape_id, + left=float(left), + top=float(top), + right=float(left + width), + bottom=float(top + height), + ) + + +def _direction_from_shape_boxes( + *, + begin_id: int | None, + end_id: int | None, + shape_boxes: dict[int, _ShapeBox] | None, +) -> str | None: + """Infer a connector direction from resolved endpoint shape centers.""" + + if begin_id is None or end_id is None or shape_boxes is None: + return None + begin_box = shape_boxes.get(begin_id) + end_box = shape_boxes.get(end_id) + if begin_box is None or end_box is None: + return None + begin_center = _shape_box_center(begin_box) + end_center = _shape_box_center(end_box) + dx = end_center[0] - begin_center[0] + dy = end_center[1] - begin_center[1] + if dx == 0 and dy == 0: + return None + angle = compute_line_angle_deg(dx, dy) + return angle_to_compass(angle) + + +def _shape_box_center(box: _ShapeBox) -> tuple[float, float]: + """Return the center point of a matched emitted shape box.""" + + return ((box.left + box.right) / 2.0, (box.top + box.bottom) / 2.0) + + +def _resolve_direct_connector_ids( + *, + connector_info: OoxmlConnectorInfo | None, + uno_connector: ConnectorSnapshot | None, + drawing_to_shape_id: dict[int, int], + shape_name_to_id: dict[str, int], +) -> tuple[int | None, int | None, bool, bool]: + """Resolve direct connector endpoints from OOXML refs and UNO shape refs.""" + + begin_id: int | None = None + end_id: int | None = None + used_ooxml_direct = False + used_uno_direct = False + if connector_info is not None: + start_id = connector_info.connection.start_drawing_id + target_id = connector_info.connection.end_drawing_id + begin_id = drawing_to_shape_id.get(start_id) if start_id is not None else None + end_id = drawing_to_shape_id.get(target_id) if target_id is not None else None + used_ooxml_direct = begin_id is not None or end_id is not None + if uno_connector is not None: + if begin_id is None and uno_connector.start_shape_name is not None: + begin_id = shape_name_to_id.get(uno_connector.start_shape_name) + used_uno_direct = used_uno_direct or begin_id is not None + if end_id is None and uno_connector.end_shape_name is not None: + end_id = shape_name_to_id.get(uno_connector.end_shape_name) + used_uno_direct = used_uno_direct or end_id is not None + return (begin_id, end_id, used_ooxml_direct, used_uno_direct) + + +def _classify_connector_resolution( + *, + begin_id: int | None, + end_id: int | None, + used_ooxml_direct: bool, + used_uno_direct: bool, + used_heuristic: bool, +) -> tuple[int | None, int | None, str, float]: + """Classify connector provenance once endpoint resolution is complete.""" + + if used_heuristic: + return (begin_id, end_id, "heuristic", 0.6) + if used_ooxml_direct and used_uno_direct: + return (begin_id, end_id, "partial", 0.9) + if used_ooxml_direct: + return (begin_id, end_id, "direct", 1.0) + if used_uno_direct: + return (begin_id, end_id, "direct", 0.9) + return (begin_id, end_id, "heuristic", 0.6) + + +class _ShapeBox: + """Axis-aligned bounding box for emitted shapes.""" + + def __init__( + self, *, shape_id: int, left: float, top: float, right: float, bottom: float + ) -> None: + """Store bounding-box coordinates for emitted shape matching.""" + + self.shape_id = shape_id + self.left = left + self.top = top + self.right = right + self.bottom = bottom diff --git a/src/exstruct/core/backends/openpyxl_backend.py b/src/exstruct/core/backends/openpyxl_backend.py index 412b5484..2d44c070 100644 --- a/src/exstruct/core/backends/openpyxl_backend.py +++ b/src/exstruct/core/backends/openpyxl_backend.py @@ -2,28 +2,59 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass import logging from pathlib import Path -from typing import Literal +import sys +from typing import Any, Literal, cast from ...models import PrintArea +from .. import cells from ..cells import ( WorkbookColorsMap, WorkbookFormulasMap, - detect_tables_openpyxl, - extract_sheet_cells, - extract_sheet_cells_with_links, - extract_sheet_colors_map, - extract_sheet_formulas_map, - extract_sheet_merged_cells, + detect_tables_openpyxl as detect_tables_openpyxl, + extract_sheet_cells as extract_sheet_cells, + extract_sheet_cells_with_links as extract_sheet_cells_with_links, + extract_sheet_colors_map as extract_sheet_colors_map, + extract_sheet_formulas_map as extract_sheet_formulas_map, + extract_sheet_merged_cells as extract_sheet_merged_cells, ) +from ..openpyxl_session import OpenpyxlExtractionSession from ..ranges import parse_range_zero_based from ..workbook import openpyxl_workbook from .base import CellData, MergedCellData, PrintAreaData logger = logging.getLogger(__name__) +_DEFAULT_OPENPYXL_WORKBOOK = openpyxl_workbook +_DEFAULT_HELPERS: dict[str, Callable[..., Any]] = { + "extract_sheet_cells": extract_sheet_cells, + "extract_sheet_cells_with_links": extract_sheet_cells_with_links, + "extract_sheet_colors_map": extract_sheet_colors_map, + "extract_sheet_formulas_map": extract_sheet_formulas_map, + "extract_sheet_merged_cells": extract_sheet_merged_cells, + "detect_tables_openpyxl": detect_tables_openpyxl, +} + + +def _helper(name: str) -> Callable[..., Any]: + """Resolve live legacy overrides, with backend aliases taking precedence.""" + current = getattr(sys.modules[__name__], name) + if current is not _DEFAULT_HELPERS[name]: + return cast(Callable[..., Any], current) + return cast(Callable[..., Any], getattr(cells, name)) + + +def _use_session(name: str, session: OpenpyxlExtractionSession | None) -> bool: + """Honor overrides on both legacy module and backend helper surfaces.""" + return ( + session is not None + and getattr(sys.modules[__name__], name) is _DEFAULT_HELPERS[name] + and getattr(cells, name) is _DEFAULT_HELPERS[name] + ) + @dataclass(frozen=True) class OpenpyxlBackend: @@ -34,6 +65,7 @@ class OpenpyxlBackend: """ file_path: Path + session: OpenpyxlExtractionSession | None = None def extract_cells(self, *, include_links: bool) -> CellData: """Extract cell rows from the workbook. @@ -44,11 +76,21 @@ def extract_cells(self, *, include_links: bool) -> CellData: Returns: Mapping of sheet name to cell rows. """ - return ( - extract_sheet_cells_with_links(self.file_path) - if include_links - else extract_sheet_cells(self.file_path) + helper = ( + "extract_sheet_cells_with_links" if include_links else "extract_sheet_cells" ) + if ( + self.session is not None + and self.file_path.suffix.lower() in {".xlsx", ".xlsm"} + and _use_session(helper, self.session) + ): + return { + ws.title: cells.extract_sheet_cells_openpyxl_ws( + ws, include_links=include_links + ) + for ws in self.session.workbook().worksheets + } + return cast(CellData, _helper(helper)(self.file_path)) def extract_print_areas(self) -> PrintAreaData: """Extract print areas per sheet using openpyxl defined names. @@ -57,6 +99,13 @@ def extract_print_areas(self) -> PrintAreaData: Mapping of sheet name to print area list. """ try: + if ( + self.session is not None + and openpyxl_workbook is _DEFAULT_OPENPYXL_WORKBOOK + ): + wb = self.session.workbook() + areas = _extract_print_areas_from_defined_names(wb) + return areas or _extract_print_areas_from_sheet_props(wb) with openpyxl_workbook( self.file_path, data_only=True, read_only=False ) as wb: @@ -80,10 +129,26 @@ def extract_colors_map( WorkbookColorsMap or None when extraction fails. """ try: - return extract_sheet_colors_map( - self.file_path, - include_default_background=include_default_background, - ignore_colors=ignore_colors, + if self.session is not None and _use_session( + "extract_sheet_colors_map", self.session + ): + return WorkbookColorsMap( + sheets={ + ws.title: cells.extract_sheet_colors_map_ws( + ws, + include_default_background=include_default_background, + ignore_colors=ignore_colors, + ) + for ws in self.session.workbook().worksheets + } + ) + return cast( + WorkbookColorsMap, + _helper("extract_sheet_colors_map")( + self.file_path, + include_default_background=include_default_background, + ignore_colors=ignore_colors, + ), ) except Exception as exc: logger.warning( @@ -98,7 +163,16 @@ def extract_merged_cells(self) -> MergedCellData: Mapping of sheet name to merged cell ranges. """ try: - return extract_sheet_merged_cells(self.file_path) + if self.session is not None and _use_session( + "extract_sheet_merged_cells", self.session + ): + return { + ws.title: cells.extract_sheet_merged_cells_ws(ws) + for ws in self.session.workbook().worksheets + } + return cast( + MergedCellData, _helper("extract_sheet_merged_cells")(self.file_path) + ) except Exception: return {} @@ -110,7 +184,19 @@ def extract_formulas_map(self) -> WorkbookFormulasMap | None: WorkbookFormulasMap | None: A mapping from sheet name to its formulas, or `None` if extraction fails. """ try: - return extract_sheet_formulas_map(self.file_path) + if self.session is not None and _use_session( + "extract_sheet_formulas_map", self.session + ): + return WorkbookFormulasMap( + sheets={ + ws.title: cells.extract_sheet_formulas_map_ws(ws) + for ws in self.session.workbook(data_only=False).worksheets + } + ) + return cast( + WorkbookFormulasMap, + _helper("extract_sheet_formulas_map")(self.file_path), + ) except Exception as exc: logger.warning( "Formula map extraction failed; skipping formulas_map. (%r)", exc @@ -135,7 +221,18 @@ def detect_tables( list[str]: Detected table candidate ranges as A1-style range strings; empty list if none are found or detection fails. """ try: - return detect_tables_openpyxl(self.file_path, sheet_name, mode=mode) + if self.session is not None and _use_session( + "detect_tables_openpyxl", self.session + ): + return cells.detect_tables_openpyxl_ws( + self.session.workbook()[sheet_name], mode=mode + ) + return cast( + list[str], + _helper("detect_tables_openpyxl")( + self.file_path, sheet_name, mode=mode + ), + ) except Exception: return [] diff --git a/src/exstruct/core/cell_types.py b/src/exstruct/core/cell_types.py new file mode 100644 index 00000000..26563e35 --- /dev/null +++ b/src/exstruct/core/cell_types.py @@ -0,0 +1,71 @@ +"""Dependency-free value contracts shared by extraction backends.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +# Use dataclasses for lightweight models +@dataclass(frozen=True) +class SheetColorsMap: + """Background color map for a single worksheet.""" + + sheet_name: str + colors_map: dict[str, list[tuple[int, int]]] + + +@dataclass(frozen=True) +class WorkbookColorsMap: + """Background color maps for all worksheets in a workbook.""" + + sheets: dict[str, SheetColorsMap] + + def get_sheet(self, sheet_name: str) -> SheetColorsMap | None: + """ + Retrieve the SheetColorsMap for a worksheet by name. + + Parameters: + sheet_name (str): Name of the worksheet to retrieve. + + Returns: + SheetColorsMap | None: The sheet's color map if present, `None` otherwise. + """ + return self.sheets.get(sheet_name) + + +@dataclass(frozen=True) +class SheetFormulasMap: + """Formula map for a single worksheet.""" + + sheet_name: str + formulas_map: dict[str, list[tuple[int, int]]] + + +@dataclass(frozen=True) +class WorkbookFormulasMap: + """Formula maps for all worksheets in a workbook.""" + + sheets: dict[str, SheetFormulasMap] + + def get_sheet(self, sheet_name: str) -> SheetFormulasMap | None: + """ + Retrieve the formulas map for a worksheet. + + Parameters: + sheet_name (str): Name of the worksheet to look up. + + Returns: + SheetFormulasMap | None: The sheet's formulas map if present, `None` if the worksheet is not found. + """ + return self.sheets.get(sheet_name) + + +@dataclass(frozen=True) +class MergedCellRange: + """Merged cell range with normalized value.""" + + r1: int + c1: int + r2: int + c2: int + v: str diff --git a/src/exstruct/core/cells.py b/src/exstruct/core/cells.py index ce35bd79..7188cd79 100644 --- a/src/exstruct/core/cells.py +++ b/src/exstruct/core/cells.py @@ -3,26 +3,43 @@ from __future__ import annotations from collections import deque -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass +from datetime import time from decimal import Decimal, InvalidOperation +from itertools import groupby import logging import math import os from pathlib import Path import re -from typing import Literal +from typing import TYPE_CHECKING, Literal import numpy as np +from openpyxl.cell.cell import Cell, MergedCell +from openpyxl.cell.read_only import EmptyCell, ReadOnlyCell from openpyxl.styles.colors import Color from openpyxl.utils import get_column_letter, range_boundaries +from openpyxl.worksheet._read_only import ReadOnlyWorksheet from openpyxl.worksheet.worksheet import Worksheet -import pandas as pd -import xlwings as xw from ..models import CellRow +from ._lazy import LazyModule +from .cell_types import ( + MergedCellRange as MergedCellRange, + SheetColorsMap as SheetColorsMap, + SheetFormulasMap as SheetFormulasMap, + WorkbookColorsMap as WorkbookColorsMap, + WorkbookFormulasMap as WorkbookFormulasMap, +) +from .openpyxl_session import OpenpyxlExtractionSession from .workbook import openpyxl_workbook +if TYPE_CHECKING: + import xlwings as xw +else: + xw = LazyModule("xlwings") + logger = logging.getLogger(__name__) _warned_keys: set[str] = set() XL_LINESTYLE_NONE = -4142 @@ -48,72 +65,6 @@ ExtractionMode = Literal["light", "libreoffice", "standard", "verbose"] -# Use dataclasses for lightweight models -@dataclass(frozen=True) -class SheetColorsMap: - """Background color map for a single worksheet.""" - - sheet_name: str - colors_map: dict[str, list[tuple[int, int]]] - - -@dataclass(frozen=True) -class WorkbookColorsMap: - """Background color maps for all worksheets in a workbook.""" - - sheets: dict[str, SheetColorsMap] - - def get_sheet(self, sheet_name: str) -> SheetColorsMap | None: - """ - Retrieve the SheetColorsMap for a worksheet by name. - - Parameters: - sheet_name (str): Name of the worksheet to retrieve. - - Returns: - SheetColorsMap | None: The sheet's color map if present, `None` otherwise. - """ - return self.sheets.get(sheet_name) - - -@dataclass(frozen=True) -class SheetFormulasMap: - """Formula map for a single worksheet.""" - - sheet_name: str - formulas_map: dict[str, list[tuple[int, int]]] - - -@dataclass(frozen=True) -class WorkbookFormulasMap: - """Formula maps for all worksheets in a workbook.""" - - sheets: dict[str, SheetFormulasMap] - - def get_sheet(self, sheet_name: str) -> SheetFormulasMap | None: - """ - Retrieve the formulas map for a worksheet. - - Parameters: - sheet_name (str): Name of the worksheet to look up. - - Returns: - SheetFormulasMap | None: The sheet's formulas map if present, `None` if the worksheet is not found. - """ - return self.sheets.get(sheet_name) - - -@dataclass(frozen=True) -class MergedCellRange: - """Merged cell range with normalized value.""" - - r1: int - c1: int - r2: int - c2: int - v: str - - @dataclass(frozen=True) class TableScanLimits: """Limits for openpyxl border scanning during table detection.""" @@ -182,8 +133,10 @@ def extract_sheet_colors_map( sheets: dict[str, SheetColorsMap] = {} with openpyxl_workbook(file_path, data_only=True, read_only=False) as wb: for ws in wb.worksheets: - sheet_map = _extract_sheet_colors( - ws, include_default_background, ignore_colors + sheet_map = extract_sheet_colors_map_ws( + ws, + include_default_background=include_default_background, + ignore_colors=ignore_colors, ) sheets[ws.title] = sheet_map return WorkbookColorsMap(sheets=sheets) @@ -202,11 +155,26 @@ def extract_sheet_formulas_map(file_path: Path) -> WorkbookFormulasMap: sheets: dict[str, SheetFormulasMap] = {} with openpyxl_workbook(file_path, data_only=False, read_only=False) as wb: for ws in wb.worksheets: - sheet_map = _extract_sheet_formulas(ws) + sheet_map = extract_sheet_formulas_map_ws(ws) sheets[ws.title] = sheet_map return WorkbookFormulasMap(sheets=sheets) +def extract_sheet_colors_map_ws( + ws: Worksheet, + *, + include_default_background: bool, + ignore_colors: set[str] | None, +) -> SheetColorsMap: + """Extract colors without reopening the worksheet's workbook.""" + return _extract_sheet_colors(ws, include_default_background, ignore_colors) + + +def extract_sheet_formulas_map_ws(ws: Worksheet) -> SheetFormulasMap: + """Extract formulas from a formula-enabled worksheet.""" + return _extract_sheet_formulas(ws) + + def extract_sheet_formulas_map_com(workbook: xw.Book) -> WorkbookFormulasMap: """ Collects and normalizes formulas from every worksheet in an xlwings workbook into per-sheet mappings. @@ -698,66 +666,174 @@ def warn_once(key: str, message: str) -> None: _warned_keys.add(key) -def extract_sheet_cells(file_path: Path) -> dict[str, list[CellRow]]: - """Read all sheets via pandas and convert to CellRow list while skipping empty cells.""" - dfs = pd.read_excel(file_path, header=None, sheet_name=None, dtype=str) - result: dict[str, list[CellRow]] = {} - for sheet_name, df in dfs.items(): - df = df.fillna("") - rows: list[CellRow] = [] - for excel_row, row in enumerate(df.itertuples(index=False, name=None), start=1): - filtered: dict[str, int | float | str] = {} - for j, v in enumerate(row): - s = "" if v is None else str(v) - if s.strip() == "": - continue - filtered[str(j)] = _coerce_numeric_preserve_format(s) - if not filtered: - continue - rows.append(CellRow(r=excel_row, c=filtered)) - result[sheet_name] = rows - return result +# Freeze the former reader's default missing-string vocabulary. Match before +# stripping: padded tokens such as " NA " are ordinary text in the old output. +_CELL_NA_STRINGS = frozenset( + { + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "", + "N/A", + "NA", + "NULL", + "NaN", + "None", + "n/a", + "nan", + "null", + } +) -def extract_sheet_cells_with_links(file_path: Path) -> dict[str, list[CellRow]]: - """ - Extract cells and hyperlinks per sheet. +def _normalize_cell_value(value: object) -> int | float | str | None: + """Preserve the former string-reader and numeric-coercion contract.""" + if value is None: + return None + text = str(value) + if text in _CELL_NA_STRINGS or not text.strip(): + return None + return _coerce_numeric_preserve_format(text) - Returns: - {sheet_name: [CellRow(r=..., c=..., links={"col_index": url, ...}), ...]} - Notes: - - Uses pandas extraction for values (same filtering as extract_sheet_cells). - - Collects hyperlinks via openpyxl (requires read_only=False because border maps/hyperlinks need full objects). - - Links are mapped by column index string (e.g., "0") to hyperlink.target. +def extract_sheet_cells_openpyxl_ws( + ws: Worksheet | ReadOnlyWorksheet, *, include_links: bool +) -> list[CellRow]: + """Extract cached values from an already open data-only worksheet. + + Hyperlinks require a non-read-only worksheet. Links on filtered cells are + retained when their row has another emitted value; link-only rows are not + invented. Worksheet ownership and closing remain with the caller. """ - cell_rows = extract_sheet_cells(file_path) - links_by_sheet: dict[str, dict[int, dict[str, str]]] = {} - with openpyxl_workbook(file_path, data_only=True, read_only=False) as wb: + cell_rows: Iterable[ + tuple[int, Iterable[tuple[int, Cell | MergedCell | ReadOnlyCell | EmptyCell]]] + ] + if isinstance(ws, Worksheet): + # iter_rows() materializes every blank in the rectangular extent. Read + # the already loaded cell store without inflating a shared sparse sheet. + cell_rows = ( + (row_number, ((coordinate[1] - 1, cell) for coordinate, cell in group)) + for row_number, group in groupby( + sorted(ws._cells.items()), key=lambda entry: entry[0][0] + ) + ) + else: + cell_rows = ( + (row_number, enumerate(row)) + for row_number, row in enumerate(ws.iter_rows(), start=1) + ) + rows: list[CellRow] = [] + for row_number, row in cell_rows: + values: dict[str, int | float | str] = {} + links: dict[str, str] = {} + for column, cell in row: + value = cell.value + if cell.data_type == "e": + value = None + elif cell.data_type == "n" and value is not None: + # Excel numeric 1.0 was converted to int before dtype=str. + integral = int(value) + value = integral if integral == value else float(value) + normalized = _normalize_cell_value(value) + if normalized is not None: + values[str(column)] = normalized + if include_links: + link = getattr(cell, "hyperlink", None) + target = getattr(link, "target", None) if link else None + if target: + links[str(column)] = target + if values: + rows.append(CellRow(r=row_number, c=values, links=links or None)) + return rows + + +def _xls_cell_value(value: object, cell_type: int, datemode: int) -> object: + """Apply the former BIFF reader's scalar conversions before normalization.""" + import xlrd + + if cell_type == xlrd.XL_CELL_ERROR: + return None + if cell_type == xlrd.XL_CELL_BOOLEAN: + return bool(value) + if cell_type == xlrd.XL_CELL_DATE: + try: + converted = xlrd.xldate.xldate_as_datetime(value, datemode) + except OverflowError: + return value + epoch_day = (1904, 1, 1) if datemode else (1899, 12, 31) + if (converted.year, converted.month, converted.day) == epoch_day: + return time( + converted.hour, + converted.minute, + converted.second, + converted.microsecond, + ) + return converted + if ( + cell_type == xlrd.XL_CELL_NUMBER + and isinstance(value, (int, float)) + and math.isfinite(value) + ): + integral = int(value) + if integral == value: + return integral + return value + + +def _extract_sheet_cells_xls(file_path: Path) -> dict[str, list[CellRow]]: + """Read legacy BIFF cached values; do not add rich XLS artifacts.""" + import xlrd + + book = xlrd.open_workbook(str(file_path), on_demand=True) + try: + result: dict[str, list[CellRow]] = {} + for sheet in book.sheets(): + rows: list[CellRow] = [] + for row_index in range(sheet.nrows): + values: dict[str, int | float | str] = {} + for column, cell in enumerate(sheet.row(row_index)): + value = _xls_cell_value(cell.value, cell.ctype, book.datemode) + normalized = _normalize_cell_value(value) + if normalized is not None: + values[str(column)] = normalized + if values: + rows.append(CellRow(r=row_index + 1, c=values)) + result[sheet.name] = rows + return result + finally: + book.release_resources() + + +def extract_sheet_cells(file_path: Path) -> dict[str, list[CellRow]]: + """Read cached values directly, preserving sheet order and sparse rows.""" + if file_path.suffix.lower() == ".xls": + return _extract_sheet_cells_xls(file_path) + with openpyxl_workbook(file_path, data_only=True, read_only=True) as wb: + result: dict[str, list[CellRow]] = {} for ws in wb.worksheets: - sheet_links: dict[int, dict[str, str]] = {} - for row in ws.iter_rows(): - for cell in row: - link = getattr(cell, "hyperlink", None) - target = getattr(link, "target", None) if link else None - if not target: - continue - col_str = str( - cell.col_idx - 1 - ) # zero-based to align with extract_sheet_cells - sheet_links.setdefault(cell.row, {})[col_str] = target - links_by_sheet[ws.title] = sheet_links - - merged: dict[str, list[CellRow]] = {} - for sheet_name, rows in cell_rows.items(): - sheet_links = links_by_sheet.get(sheet_name, {}) - merged_rows: list[CellRow] = [] - for row in rows: - links = sheet_links.get(row.r, {}) - merged_rows.append(CellRow(r=row.r, c=row.c, links=links or None)) - merged[sheet_name] = merged_rows - wb.close() - return merged + # Like the old reader, ignore potentially incorrect declared bounds. + ws.reset_dimensions() + result[ws.title] = extract_sheet_cells_openpyxl_ws(ws, include_links=False) + return result + + +def extract_sheet_cells_with_links(file_path: Path) -> dict[str, list[CellRow]]: + """Extract cells and hyperlink targets with a single workbook lifetime.""" + if file_path.suffix.lower() == ".xls": + # XLS hyperlinks were never supported by the openpyxl link reader. + return extract_sheet_cells(file_path) + with openpyxl_workbook(file_path, data_only=True, read_only=False) as wb: + return { + ws.title: extract_sheet_cells_openpyxl_ws(ws, include_links=True) + for ws in wb.worksheets + } def extract_sheet_merged_cells(file_path: Path) -> dict[str, list[MergedCellRange]]: @@ -769,32 +845,33 @@ def extract_sheet_merged_cells(file_path: Path) -> dict[str, list[MergedCellRang Returns: Mapping of sheet name to merged cell ranges. """ - merged_by_sheet: dict[str, list[MergedCellRange]] = {} with openpyxl_workbook(file_path, data_only=True, read_only=False) as wb: - for ws in wb.worksheets: - merged_ranges = getattr(ws, "merged_cells", None) - if merged_ranges is None: - merged_by_sheet[ws.title] = [] - continue - results: list[MergedCellRange] = [] - for merged_range in getattr(merged_ranges, "ranges", []): - bounds = range_boundaries(str(merged_range)) - min_col, min_row, max_col, max_row = bounds - cell_value = ws.cell(row=min_row, column=min_col).value - value_str = "" if cell_value is None else str(cell_value) - if value_str == "": - value_str = " " - results.append( - MergedCellRange( - r1=min_row, - c1=min_col - 1, - r2=max_row, - c2=max_col - 1, - v=value_str, - ) - ) - merged_by_sheet[ws.title] = results - return merged_by_sheet + return {ws.title: extract_sheet_merged_cells_ws(ws) for ws in wb.worksheets} + + +def extract_sheet_merged_cells_ws(ws: Worksheet) -> list[MergedCellRange]: + """Extract merged ranges from an already owned worksheet.""" + merged_ranges = getattr(ws, "merged_cells", None) + if merged_ranges is None: + return [] + results: list[MergedCellRange] = [] + for merged_range in getattr(merged_ranges, "ranges", []): + bounds = range_boundaries(str(merged_range)) + min_col, min_row, max_col, max_row = bounds + cell_value = ws.cell(row=min_row, column=min_col).value + value_str = "" if cell_value is None else str(cell_value) + if value_str == "": + value_str = " " + results.append( + MergedCellRange( + r1=min_row, + c1=min_col - 1, + r2=max_row, + c2=max_col - 1, + v=value_str, + ) + ) + return results def shrink_to_content( # noqa: C901 @@ -957,19 +1034,19 @@ def load_border_maps_xlsx( # noqa: C901 with openpyxl_workbook(xlsx_path, data_only=True, read_only=False) as wb: if sheet_name not in wb.sheetnames: raise KeyError(f"Sheet '{sheet_name}' not found in {xlsx_path}") + return load_border_maps_openpyxl_ws(wb[sheet_name], scan_limits=scan_limits) - ws = wb[sheet_name] - try: - min_col, min_row, max_col, max_row = range_boundaries( - ws.calculate_dimension() - ) - except Exception: - min_col, min_row, max_col, max_row = ( - 1, - 1, - ws.max_column or 1, - ws.max_row or 1, - ) + +def load_border_maps_openpyxl_ws( # noqa: C901 + ws: Worksheet, + *, + scan_limits: TableScanLimits | None = None, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, int, int]: + """Load border maps without reopening the worksheet's workbook.""" + try: + min_col, min_row, max_col, max_row = range_boundaries(ws.calculate_dimension()) + except Exception: + min_col, min_row, max_col, max_row = (1, 1, ws.max_column or 1, ws.max_row or 1) resolved_limits = scan_limits or _DEFAULT_TABLE_SCAN_LIMITS scan_max_row = min(max_row, resolved_limits.max_rows) @@ -1120,6 +1197,47 @@ def _detect_border_clusters_python( return rects +def _detect_border_clusters_sparse( + has_border: np.ndarray, min_size: int +) -> list[tuple[int, int, int, int]]: + """Evaluate set-based BFS without changing backend selection. + + Seeds follow NumPy's row-major nonzero order, matching the existing BFS + and SciPy component order. Only occupied coordinates enter the set; + the input grid is still scanned by nonzero. Bounding boxes accumulate + incrementally instead of retaining every visited row and column. + """ + rows, columns = np.nonzero(has_border) + seeds = list(zip(rows.tolist(), columns.tolist(), strict=True)) + remaining = set(seeds) + rects: list[tuple[int, int, int, int]] = [] + for seed in seeds: + if seed not in remaining: + continue + remaining.remove(seed) + queue = deque([seed]) + min_row = max_row = seed[0] + min_col = max_col = seed[1] + size = 0 + while queue: + row, col = queue.popleft() + size += 1 + min_row, max_row = min(min_row, row), max(max_row, row) + min_col, max_col = min(min_col, col), max(max_col, col) + for neighbor in ( + (row + 1, col), + (row - 1, col), + (row, col + 1), + (row, col - 1), + ): + if neighbor in remaining: + remaining.remove(neighbor) + queue.append(neighbor) + if size >= min_size: + rects.append((min_row, min_col, max_row, max_col)) + return rects + + def _resolve_border_cluster_backend() -> Literal["auto", "python", "numpy"]: """Resolve the border clustering backend from environment.""" value = os.getenv(_BORDER_CLUSTER_BACKEND_ENV, "").strip().lower() @@ -1850,51 +1968,69 @@ def detect_tables_openpyxl( scan_limits: TableScanLimits | None = None, ) -> list[str]: """Detect table-like ranges via openpyxl tables and border clusters.""" - resolved_limits = _resolve_table_scan_limits(mode, scan_limits) with openpyxl_workbook(xlsx_path, data_only=True, read_only=False) as wb: - ws = wb[sheet_name] - tables = _extract_openpyxl_table_refs(ws) - - has_border, top_edge, bottom_edge, left_edge, right_edge, max_row, max_col = ( - load_border_maps_xlsx( - xlsx_path, - sheet_name, - scan_limits=resolved_limits, - ) + return detect_tables_openpyxl_ws( + wb[sheet_name], mode=mode, scan_limits=scan_limits ) - rects = _detect_border_rectangles(has_border, min_size=4) - merged_rects = _merge_rectangles(rects) - dedup: set[str] = set(tables) - for top_row, left_col, bottom_row, right_col in merged_rects: - top_row, left_col, bottom_row, right_col = shrink_to_content_openpyxl( - ws, - top_row, - left_col, - bottom_row, - right_col, - require_inside_border=False, - top_edge=top_edge, - bottom_edge=bottom_edge, - left_edge=left_edge, - right_edge=right_edge, - min_nonempty_ratio=0.0, - ) - vals_block = _get_values_block(ws, top_row, left_col, bottom_row, right_col) - candidates = _collect_table_candidates_from_values( - _normalize_matrix(vals_block), - base_top=top_row, - base_left=left_col, - col_name=get_column_letter, - ) - for addr in candidates: - if addr not in dedup: - dedup.add(addr) - tables.append(addr) - return tables +def detect_tables_openpyxl_ws( + ws: Worksheet, + *, + mode: ExtractionMode = "standard", + scan_limits: TableScanLimits | None = None, +) -> list[str]: + """Detect tables using an already loaded worksheet.""" + resolved_limits = _resolve_table_scan_limits(mode, scan_limits) + tables = _extract_openpyxl_table_refs(ws) + + has_border, top_edge, bottom_edge, left_edge, right_edge, max_row, max_col = ( + load_border_maps_openpyxl_ws( + ws, + scan_limits=resolved_limits, + ) + ) + rects = _detect_border_rectangles(has_border, min_size=4) + merged_rects = _merge_rectangles(rects) + dedup: set[str] = set(tables) + + for top_row, left_col, bottom_row, right_col in merged_rects: + top_row, left_col, bottom_row, right_col = shrink_to_content_openpyxl( + ws, + top_row, + left_col, + bottom_row, + right_col, + require_inside_border=False, + top_edge=top_edge, + bottom_edge=bottom_edge, + left_edge=left_edge, + right_edge=right_edge, + min_nonempty_ratio=0.0, + ) + vals_block = _get_values_block(ws, top_row, left_col, bottom_row, right_col) + candidates = _collect_table_candidates_from_values( + _normalize_matrix(vals_block), + base_top=top_row, + base_left=left_col, + col_name=get_column_letter, + ) + for addr in candidates: + if addr not in dedup: + dedup.add(addr) + tables.append(addr) + return tables + + +_DEFAULT_DETECT_TABLES_OPENPYXL = detect_tables_openpyxl -def detect_tables(sheet: xw.Sheet, *, mode: ExtractionMode = "standard") -> list[str]: + +def detect_tables( + sheet: xw.Sheet, + *, + mode: ExtractionMode = "standard", + openpyxl_session: OpenpyxlExtractionSession | None = None, +) -> list[str]: """Detect table-like ranges with COM and optional openpyxl fallback. Args: @@ -1928,6 +2064,14 @@ def detect_tables(sheet: xw.Sheet, *, mode: ExtractionMode = "standard") -> list return detect_tables_xlwings(sheet) try: + if ( + openpyxl_session is not None + and openpyxl_session.file_path.resolve() == excel_path.resolve() + and detect_tables_openpyxl is _DEFAULT_DETECT_TABLES_OPENPYXL + ): + return detect_tables_openpyxl_ws( + openpyxl_session.workbook()[sheet.name], mode=mode + ) return detect_tables_openpyxl(excel_path, sheet.name, mode=mode) except Exception as e: warn_once( diff --git a/src/exstruct/core/drawing_geometry.py b/src/exstruct/core/drawing_geometry.py new file mode 100644 index 00000000..90b1b85e --- /dev/null +++ b/src/exstruct/core/drawing_geometry.py @@ -0,0 +1,37 @@ +"""Pure connector angle helpers shared by OOXML and COM shapes.""" + +import math +from typing import Literal, cast + + +def compute_line_angle_deg(w: float, h: float) -> float: + """ + Compute the clockwise angle (in degrees) in Excel coordinates where 0° points East. + + Parameters: + w (float): Horizontal delta (width, positive to the right). + h (float): Vertical delta (height, positive downward). + + Returns: + float: Angle in degrees measured clockwise from East (e.g., 0° = East, 90° = South). + """ + return math.degrees(math.atan2(h, w)) % 360.0 + + +def angle_to_compass( + angle: float, +) -> Literal["E", "SE", "S", "SW", "W", "NW", "N", "NE"]: + """ + Map an angle in degrees to one of eight compass directions. + + The angle is interpreted with 0 degrees at East and increasing values rotating counterclockwise (45 -> NE, 90 -> N). + + Parameters: + angle (float): Angle in degrees. + + Returns: + str: One of `"E"`, `"SE"`, `"S"`, `"SW"`, `"W"`, `"NW"`, `"N"`, or `"NE"` corresponding to the nearest 8-point compass direction. + """ + dirs = ["E", "NE", "N", "NW", "W", "SW", "S", "SE"] + idx = int(((angle + 22.5) % 360) // 45) + return cast(Literal["E", "SE", "S", "SW", "W", "NW", "N", "NE"], dirs[idx]) diff --git a/src/exstruct/core/openpyxl_session.py b/src/exstruct/core/openpyxl_session.py new file mode 100644 index 00000000..418516cd --- /dev/null +++ b/src/exstruct/core/openpyxl_session.py @@ -0,0 +1,50 @@ +"""Explicit, extraction-scoped ownership of lazy openpyxl workbooks.""" + +from __future__ import annotations + +from contextlib import ExitStack +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openpyxl.workbook.workbook import Workbook + +from . import workbook + + +class OpenpyxlExtractionSession: + """Share compatible workbook variants until final model construction ends.""" + + def __init__(self, file_path: Path) -> None: + self.file_path = file_path + self._stack = ExitStack() + self._workbooks: dict[bool, Workbook] = {} + self._closed = False + + def __enter__(self) -> OpenpyxlExtractionSession: + if self._closed: + raise RuntimeError("Openpyxl extraction session is closed") + return self + + def workbook(self, *, data_only: bool = True) -> Workbook: + """Load each regular worksheet-capable variant only on first use.""" + if self._closed: + raise RuntimeError("Openpyxl extraction session is closed") + if data_only not in self._workbooks: + self._workbooks[data_only] = self._stack.enter_context( + workbook.openpyxl_workbook( + self.file_path, data_only=data_only, read_only=False + ) + ) + return self._workbooks[data_only] + + def close(self) -> None: + """Release all successfully loaded variants, including after errors.""" + if self._closed: + return + self._closed = True + self._workbooks.clear() + self._stack.close() + + def __exit__(self, *exc_info: object) -> None: + self.close() diff --git a/src/exstruct/core/pipeline.py b/src/exstruct/core/pipeline.py index 70f05227..8ffddf06 100644 --- a/src/exstruct/core/pipeline.py +++ b/src/exstruct/core/pipeline.py @@ -4,13 +4,13 @@ from collections.abc import Callable, Sequence from dataclasses import dataclass, field +from inspect import unwrap import logging import os from pathlib import Path +import sys import time -from typing import Literal - -import xlwings as xw +from typing import TYPE_CHECKING, Any, Literal, overload from ..errors import FallbackReason from ..models import ( @@ -22,9 +22,8 @@ SmartArt, WorkbookData, ) +from ._lazy import LazyModule from .backends.base import RichBackend -from .backends.com_backend import ComBackend, ComRichBackend -from .backends.libreoffice_backend import LibreOfficeRichBackend from .backends.ooxml_backend import OoxmlRichBackend from .backends.openpyxl_backend import OpenpyxlBackend from .cells import ( @@ -34,13 +33,68 @@ detect_tables, warn_once, ) -from .charts import get_charts from .libreoffice import LibreOfficeUnavailableError from .logging_utils import log_fallback from .modeling import SheetRawData, WorkbookRawData, build_workbook_data -from .shapes import get_shapes_with_position +from .openpyxl_session import OpenpyxlExtractionSession from .workbook import xlwings_workbook +if TYPE_CHECKING: + import xlwings as xw + + from .backends.com_backend import ( + ComBackend as ComBackend, + ComRichBackend as ComRichBackend, + ) + from .backends.libreoffice_backend import ( + LibreOfficeRichBackend as LibreOfficeRichBackend, + ) +else: + xw = LazyModule("xlwings") + + +def __getattr__(name: str) -> Any: # noqa: ANN401 - heterogeneous compatibility exports + """Resolve backend compatibility exports only when explicitly requested.""" + from . import backends + + if name in {"ComBackend", "ComRichBackend", "LibreOfficeRichBackend"}: + return getattr(backends, name) + raise AttributeError(name) + + +@overload +def _backend_type(name: Literal["ComBackend"]) -> type[ComBackend]: ... + + +@overload +def _backend_type(name: Literal["ComRichBackend"]) -> type[ComRichBackend]: ... + + +@overload +def _backend_type( + name: Literal["LibreOfficeRichBackend"], +) -> type[LibreOfficeRichBackend]: ... + + +def _backend_type(name: str) -> Any: # noqa: ANN401 - heterogeneous compatibility exports + """Honor explicitly overridden legacy aliases before lazy resolution.""" + return getattr(sys.modules[__name__], name) + + +def get_shapes_with_position(workbook: xw.Book, mode: str = "standard") -> ShapeData: + """Forward to the live COM shape implementation, preserving overrides.""" + from . import shapes + + return shapes.get_shapes_with_position(workbook, mode=mode) + + +def get_charts(sheet: xw.Sheet, mode: ExtractionMode = "standard") -> list[Chart]: + """Forward to the live COM chart implementation, preserving overrides.""" + from . import charts + + return charts.get_charts(sheet, mode=mode) + + ExtractionMode = Literal["light", "libreoffice", "standard", "verbose"] CellData = dict[str, list[CellRow]] PrintAreaData = dict[str, list[PrintArea]] @@ -107,10 +161,13 @@ class ExtractionArtifacts: shape_data: ShapeData = field(default_factory=dict) chart_data: ChartData = field(default_factory=dict) merged_cell_data: MergedCellData = field(default_factory=dict) + openpyxl_session: OpenpyxlExtractionSession | None = field( + default=None, repr=False, compare=False + ) ExtractionStep = Callable[[ExtractionInputs, ExtractionArtifacts], None] -ComExtractionStep = Callable[[ExtractionInputs, ExtractionArtifacts, xw.Book], None] +ComExtractionStep = Callable[[ExtractionInputs, ExtractionArtifacts, "xw.Book"], None] @dataclass(frozen=True) @@ -577,7 +634,7 @@ def step_extract_cells( inputs: Pipeline inputs. artifacts: Artifact container to update. """ - backend = OpenpyxlBackend(inputs.file_path) + backend = OpenpyxlBackend(inputs.file_path, session=artifacts.openpyxl_session) artifacts.cell_data = backend.extract_cells(include_links=inputs.include_cell_links) @@ -591,7 +648,7 @@ def step_extract_print_areas_openpyxl( inputs (ExtractionInputs): Pipeline inputs containing the file path and extraction options. artifacts (ExtractionArtifacts): Mutable artifact container; `artifacts.print_area_data` will be set to the extracted print area mapping. """ - backend = OpenpyxlBackend(inputs.file_path) + backend = OpenpyxlBackend(inputs.file_path, session=artifacts.openpyxl_session) artifacts.print_area_data = backend.extract_print_areas() @@ -607,7 +664,7 @@ def step_extract_formulas_map_openpyxl( inputs (ExtractionInputs): Resolved pipeline inputs (provides file_path). artifacts (ExtractionArtifacts): Mutable container to receive the extracted formulas map. """ - backend = OpenpyxlBackend(inputs.file_path) + backend = OpenpyxlBackend(inputs.file_path, session=artifacts.openpyxl_session) try: artifacts.formulas_map_data = backend.extract_formulas_map() except Exception as exc: @@ -626,7 +683,7 @@ def step_extract_colors_map_openpyxl( Sets artifacts.colors_map_data to the colors map extracted from inputs.file_path, respecting inputs.include_default_background and inputs.ignore_colors. """ - backend = OpenpyxlBackend(inputs.file_path) + backend = OpenpyxlBackend(inputs.file_path, session=artifacts.openpyxl_session) artifacts.colors_map_data = backend.extract_colors_map( include_default_background=inputs.include_default_background, ignore_colors=inputs.ignore_colors, @@ -642,7 +699,7 @@ def step_extract_merged_cells_openpyxl( inputs: Pipeline inputs. artifacts: Artifact container to update. """ - backend = OpenpyxlBackend(inputs.file_path) + backend = OpenpyxlBackend(inputs.file_path, session=artifacts.openpyxl_session) artifacts.merged_cell_data = backend.extract_merged_cells() @@ -685,9 +742,12 @@ def step_extract_print_areas_com( artifacts: Artifact container to update. workbook: xlwings workbook instance. """ + + backend_type = _backend_type("ComBackend") + if artifacts.print_area_data: return - artifacts.print_area_data = ComBackend(workbook).extract_print_areas() + artifacts.print_area_data = backend_type(workbook).extract_print_areas() def step_extract_auto_page_breaks_com( @@ -701,7 +761,10 @@ def step_extract_auto_page_breaks_com( artifacts (ExtractionArtifacts): Mutable artifact container; updated with extracted data. workbook (xw.Book): xlwings COM workbook used to read auto page break settings. """ - artifacts.auto_page_break_data = ComBackend(workbook).extract_auto_page_breaks() + + backend_type = _backend_type("ComBackend") + + artifacts.auto_page_break_data = backend_type(workbook).extract_auto_page_breaks() def step_extract_formulas_map_com( @@ -716,8 +779,11 @@ def step_extract_formulas_map_com( Parameters: workbook (xlwings.Book): COM workbook to extract formulas from. """ + + backend_type = _backend_type("ComBackend") + try: - artifacts.formulas_map_data = ComBackend(workbook).extract_formulas_map() + artifacts.formulas_map_data = backend_type(workbook).extract_formulas_map() except Exception as exc: logger.warning( "Failed to extract formulas_map via COM. (%r)", @@ -735,7 +801,10 @@ def step_extract_colors_map_com( artifacts: Artifact container to update. workbook: xlwings workbook instance. """ - com_result = ComBackend(workbook).extract_colors_map( + + backend_type = _backend_type("ComBackend") + + com_result = backend_type(workbook).extract_colors_map( include_default_background=inputs.include_default_background, ignore_colors=inputs.ignore_colors, ) @@ -744,7 +813,7 @@ def step_extract_colors_map_com( return if artifacts.colors_map_data is None: artifacts.colors_map_data = OpenpyxlBackend( - inputs.file_path + inputs.file_path, session=artifacts.openpyxl_session ).extract_colors_map( include_default_background=inputs.include_default_background, ignore_colors=inputs.ignore_colors, @@ -899,6 +968,9 @@ def _safe_col_index(col_key: str) -> int | None: return None +_DEFAULT_DETECT_TABLES = detect_tables + + def collect_sheet_raw_data( *, cell_data: CellData, @@ -912,6 +984,7 @@ def collect_sheet_raw_data( auto_page_break_data: PrintAreaData | None = None, formulas_map_data: WorkbookFormulasMap | None = None, colors_map_data: WorkbookColorsMap | None = None, + openpyxl_session: OpenpyxlExtractionSession | None = None, ) -> dict[str, SheetRawData]: """ Collect per-sheet raw extraction data and assemble SheetRawData for each sheet. @@ -947,7 +1020,12 @@ def collect_sheet_raw_data( rows=filtered_rows, shapes=shape_data.get(sheet_name, []), charts=chart_data.get(sheet_name, []), - table_candidates=detect_tables(sheet, mode=mode), + table_candidates=( + detect_tables(sheet, mode=mode, openpyxl_session=openpyxl_session) + if openpyxl_session is not None + and unwrap(detect_tables) is _DEFAULT_DETECT_TABLES + else detect_tables(sheet, mode=mode) + ), print_areas=print_area_data.get(sheet_name, []) if print_area_data else [], auto_print_areas=auto_page_break_data.get(sheet_name, []) if auto_page_break_data @@ -966,13 +1044,14 @@ def resolve_rich_backend( workbook: xw.Book | None = None, ) -> RichBackend: """Resolve the rich extraction backend for the requested mode.""" + if inputs.mode == "light": return OoxmlRichBackend(inputs.file_path) if inputs.mode == "libreoffice": - return LibreOfficeRichBackend(inputs.file_path) + return _backend_type("LibreOfficeRichBackend")(inputs.file_path) if workbook is None: raise ValueError("COM workbook is required for COM-backed rich extraction.") - return ComRichBackend(workbook) + return _backend_type("ComRichBackend")(workbook) def _run_light_pipeline( @@ -1088,6 +1167,20 @@ def _run_libreoffice_pipeline( def run_extraction_pipeline(inputs: ExtractionInputs) -> PipelineResult: + """Own workbook resources across presteps, COM/fallback and model building.""" + session = OpenpyxlExtractionSession(inputs.file_path) + try: + with session: + return _run_extraction_pipeline(inputs, session) + finally: + # Context managers do not invoke __exit__ when __enter__ raises. + session.close() + + +def _run_extraction_pipeline( + inputs: ExtractionInputs, + session: OpenpyxlExtractionSession, +) -> PipelineResult: """ Execute the configured extraction pipeline and produce the extraction result. @@ -1098,7 +1191,9 @@ def run_extraction_pipeline(inputs: ExtractionInputs) -> PipelineResult: PipelineResult: Contains the constructed workbook data, collected artifacts, and pipeline execution state (including COM attempt/success and any fallback reason). """ plan = build_pipeline_plan(inputs) - artifacts = run_pipeline(plan.pre_com_steps, inputs, ExtractionArtifacts()) + artifacts = run_pipeline( + plan.pre_com_steps, inputs, ExtractionArtifacts(openpyxl_session=session) + ) state = PipelineState() def _fallback( @@ -1178,6 +1273,7 @@ def _fallback( else None, formulas_map_data=artifacts.formulas_map_data, colors_map_data=artifacts.colors_map_data, + openpyxl_session=session, ) raw_workbook = WorkbookRawData( book_name=inputs.file_path.name, sheets=raw_sheets @@ -1219,7 +1315,7 @@ def build_cells_tables_workbook( WorkbookData: A workbook composed from the available per-sheet cell rows, detected table candidates, merged-cell information, and any resolved formulas and colors maps. When `include_rich_artifacts` is false, shapes and charts are empty. Formulas and colors maps are extracted from artifacts or from the Openpyxl backend when requested and not already present. """ logger.info("Building fallback workbook: %s", reason) - backend = OpenpyxlBackend(inputs.file_path) + backend = OpenpyxlBackend(inputs.file_path, session=artifacts.openpyxl_session) colors_map_data = artifacts.colors_map_data if inputs.include_colors_map and colors_map_data is None: colors_map_data = backend.extract_colors_map( diff --git a/src/exstruct/core/shapes.py b/src/exstruct/core/shapes.py index e8de436a..40f5c4f2 100644 --- a/src/exstruct/core/shapes.py +++ b/src/exstruct/core/shapes.py @@ -3,47 +3,17 @@ from __future__ import annotations from collections.abc import Callable, Iterable, Iterator -import math -from typing import Literal, Protocol, SupportsInt, cast, runtime_checkable +from typing import Protocol, SupportsInt, cast, runtime_checkable import xlwings as xw from xlwings import Book from ..models import Arrow, Shape, SmartArt, SmartArtNode from ..models.maps import MSO_AUTO_SHAPE_TYPE_MAP, MSO_SHAPE_TYPE_MAP - - -def compute_line_angle_deg(w: float, h: float) -> float: - """ - Compute the clockwise angle (in degrees) in Excel coordinates where 0° points East. - - Parameters: - w (float): Horizontal delta (width, positive to the right). - h (float): Vertical delta (height, positive downward). - - Returns: - float: Angle in degrees measured clockwise from East (e.g., 0° = East, 90° = South). - """ - return math.degrees(math.atan2(h, w)) % 360.0 - - -def angle_to_compass( - angle: float, -) -> Literal["E", "SE", "S", "SW", "W", "NW", "N", "NE"]: - """ - Map an angle in degrees to one of eight compass directions. - - The angle is interpreted with 0 degrees at East and increasing values rotating counterclockwise (45 -> NE, 90 -> N). - - Parameters: - angle (float): Angle in degrees. - - Returns: - str: One of `"E"`, `"SE"`, `"S"`, `"SW"`, `"W"`, `"NW"`, `"N"`, or `"NE"` corresponding to the nearest 8-point compass direction. - """ - dirs = ["E", "NE", "N", "NW", "W", "SW", "S", "SE"] - idx = int(((angle + 22.5) % 360) // 45) - return cast(Literal["E", "SE", "S", "SW", "W", "NW", "N", "NE"], dirs[idx]) +from .drawing_geometry import ( + angle_to_compass as angle_to_compass, + compute_line_angle_deg as compute_line_angle_deg, +) def coord_to_cell_by_edges( diff --git a/src/exstruct/core/workbook.py b/src/exstruct/core/workbook.py index 3aa9430e..7d25fff3 100644 --- a/src/exstruct/core/workbook.py +++ b/src/exstruct/core/workbook.py @@ -4,11 +4,23 @@ from contextlib import contextmanager import logging from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import warnings -from openpyxl import load_workbook -import xlwings as xw +from ._lazy import LazyModule + +if TYPE_CHECKING: + import xlwings as xw +else: + xw = LazyModule("xlwings") + + +def load_workbook(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 - preserve loader signature + """Resolve the live openpyxl loader, preserving the legacy override surface.""" + from openpyxl import load_workbook as loader + + return loader(*args, **kwargs) + logger = logging.getLogger(__name__) diff --git a/tests/benchmark/test_performance.py b/tests/benchmark/test_performance.py index dee5faa8..b38afe17 100644 --- a/tests/benchmark/test_performance.py +++ b/tests/benchmark/test_performance.py @@ -118,8 +118,10 @@ def test_fresh_worker_measures_light_and_standard_fallback( assert stages["table_detection"]["calls"] == 1 assert stages["model_construction"]["calls"] == 1 assert stages["formula_extraction"]["calls"] == 0 - assert result["profile"]["openpyxl_workbook_opens"] >= 3 - assert result["profile"]["archive_opens"] >= 3 + assert result["profile"]["openpyxl_workbook_opens"] == 1 + # OOXML drawings use separate archives; workbook parses and ZIP opens + # intentionally measure different resource types. + assert result["profile"]["archive_opens"] >= 1 state = result["profile"]["pipeline_state"] assert state["com_succeeded"] is False assert state["fallback_reason"] == ( @@ -176,6 +178,7 @@ def test_optional_profile_and_supervisor_report( assert len(result["input"]["sha256"]) == 64 assert result["environment"]["SKIP_COM_TESTS"] == "1" assert result["profile"]["all_features"] is True + assert result["profile"]["openpyxl_workbook_opens"] == 2 for stage in ("formula_extraction", "color_extraction", "merged_cell_extraction"): assert result["profile"]["stages"][stage]["calls"] == 1 assert result["startup"]["cold_extraction_process_ms"] > 0 diff --git a/tests/core/cells_reader_expected.json b/tests/core/cells_reader_expected.json new file mode 100644 index 00000000..c9839066 --- /dev/null +++ b/tests/core/cells_reader_expected.json @@ -0,0 +1,751 @@ +{ + "xlsx": { + "Values": [ + { + "r": 1, + "c": { + "0": 123, + "1": 1.5, + "2": 123, + "3": 1.5, + "4": "True", + "5": "False", + "7": " x " + }, + "links": null + }, + { + "r": 2, + "c": { + "0": 0, + "1": -5, + "2": 100000000000000000000, + "3": "1e2", + "4": 1, + "5": -0.5, + "6": "1.", + "7": " NA " + }, + "links": null + }, + { + "r": 3, + "c": { + "0": "2020-01-02 00:00:00", + "1": "2020-01-02 03:04:05.123000", + "2": "03:04:05.123000", + "3": "2 days, 3:00:00" + }, + "links": null + }, + { + "r": 9, + "c": { + "3": "sparse" + }, + "links": null + }, + { + "r": 10, + "c": { + "0": 2, + "1": "True", + "2": "cached" + }, + "links": null + }, + { + "r": 11, + "c": { + "0": "visible", + "2": "internal" + }, + "links": null + }, + { + "r": 14, + "c": { + "0": "first" + }, + "links": null + } + ], + "Empty": [], + "Last": [ + { + "r": 3, + "c": { + "2": "last" + }, + "links": null + } + ] + }, + "links": { + "Values": [ + { + "r": 1, + "c": { + "0": 123, + "1": 1.5, + "2": 123, + "3": 1.5, + "4": "True", + "5": "False", + "7": " x " + }, + "links": null + }, + { + "r": 2, + "c": { + "0": 0, + "1": -5, + "2": 100000000000000000000, + "3": "1e2", + "4": 1, + "5": -0.5, + "6": "1.", + "7": " NA " + }, + "links": null + }, + { + "r": 3, + "c": { + "0": "2020-01-02 00:00:00", + "1": "2020-01-02 03:04:05.123000", + "2": "03:04:05.123000", + "3": "2 days, 3:00:00" + }, + "links": null + }, + { + "r": 9, + "c": { + "3": "sparse" + }, + "links": null + }, + { + "r": 10, + "c": { + "0": 2, + "1": "True", + "2": "cached" + }, + "links": null + }, + { + "r": 11, + "c": { + "0": "visible", + "2": "internal" + }, + "links": { + "1": "https://example.test/na" + } + }, + { + "r": 14, + "c": { + "0": "first" + }, + "links": null + } + ], + "Empty": [], + "Last": [ + { + "r": 3, + "c": { + "2": "last" + }, + "links": null + } + ] + }, + "xls": { + "表": [ + { + "r": 1, + "c": { + "0": "様式第１号（第２条関係）" + }, + "links": null + }, + { + "r": 2, + "c": { + "5": "自動車臨時運行許可申請書" + }, + "links": null + }, + { + "r": 3, + "c": { + "5": "APPLICATION FOR CAR TEMPORARY PLATE" + }, + "links": null + }, + { + "r": 4, + "c": { + "2": "※注：裏面をよく読んで太線内を記入し、 必要な書類を添えて提出してください。" + }, + "links": null + }, + { + "r": 5, + "c": { + "0": "車名\nMaker of the vehicle", + "2": "トヨタ・ホンダ・ニッサン・スズキ・マツダ・ダイハツ・三菱・その他（　　　　　　　　）", + "9": "自動車損害賠償責任保険　Car Insurance " + }, + "links": null + }, + { + "r": 7, + "c": { + "0": "形状\nType of Body", + "2": "１　箱形（Box-shaped） ２　ステーションワゴン（Station Wagon） ", + "9": "保険会社名 \nName of Co.", + "11": "１．損害保険ジャパン株式会社\n２．東京海上日動火災保険株式会社\n３．あいおいニッセイ同和損害保険株式会社\n４．全国共済農業協同組合連合会\n５．三井住友海上火災保険株式会社\n６．その他（　　　　　　　　　　　　　）" + }, + "links": null + }, + { + "r": 8, + "c": { + "2": "３　バン（Van） ４　キャブオーバー（Cab-over） " + }, + "links": null + }, + { + "r": 9, + "c": { + "2": "５　オートバイ （ motorcycle ） ６　その他( 　　　　　　　　　 ) " + }, + "links": null + }, + { + "r": 10, + "c": { + "0": "車台番号\nSerial No. " + }, + "links": null + }, + { + "r": 11, + "c": { + "0": "運行の目的\nPurpose", + "2": "１　車検のための回送( Inspection )　 ２　登録のための回送( Registration ) " + }, + "links": null + }, + { + "r": 12, + "c": { + "2": "３　封印取付け( Seal )のための回送 " + }, + "links": null + }, + { + "r": 13, + "c": { + "9": "証明書番号\nVoucher No." + }, + "links": null + }, + { + "r": 14, + "c": { + "2": "４　その他 （other） （ ） " + }, + "links": null + }, + { + "r": 15, + "c": { + "0": "運行の経路\nRoute ", + "2": "※発着主要経路の地点名を記入してください。\n出発地（From）　　　　　　　　　経由地（Via）　　　　　　　　　到着地（To） \n\n　　　　　　　　～　　　　　　　　　　　　　　　　　　～", + "9": "保険期間\nInsurannce\nPeriod ", + "11": "自(From) ", + "15": "年", + "16": "月", + "17": "日" + }, + "links": null + }, + { + "r": 16, + "c": { + "11": "至(To) ", + "15": "年", + "16": "月", + "17": "日" + }, + "links": null + }, + { + "r": 17, + "c": { + "9": "備 考 " + }, + "links": null + }, + { + "r": 18, + "c": { + "0": "運行の期間\nService period", + "2": "自(From) 年 月 日 ～ \n至(To) 年 月 日 （ 日間） " + }, + "links": null + }, + { + "r": 19, + "c": { + "2": "※目的達成に必要な最小限の日数を記入してください。 " + }, + "links": null + }, + { + "r": 20, + "c": { + "2": "（通常、整備のための回送は１日間、車検・登録のための回送は、１～２日間です。) " + }, + "links": null + }, + { + "r": 21, + "c": { + "1": "裏面の注意事項に同意の上、上記のとおり臨時運行の許可を申請します。" + }, + "links": null + }, + { + "r": 22, + "c": { + "12": "年 月 日" + }, + "links": null + }, + { + "r": 24, + "c": { + "4": "湖南市長宛" + }, + "links": null + }, + { + "r": 25, + "c": { + "9": "番号標番号 ", + "11": " 枚数 " + }, + "links": null + }, + { + "r": 26, + "c": { + "0": "申 請 人", + "1": "住所\nApplicant's Address" + }, + "links": null + }, + { + "r": 27, + "c": { + "11": "滋　　 　 　― 　　　 １・２ " + }, + "links": null + }, + { + "r": 28, + "c": { + "1": "氏名又は名称\nName\n※法人の場合は\n代表者名も\n記入してください。", + "9": "許可番号 ", + "11": "№ " + }, + "links": null + }, + { + "r": 31, + "c": { + "3": "（代表者） ", + "9": "許可年月日", + "11": "年 月 日 " + }, + "links": null + }, + { + "r": 35, + "c": { + "3": "電話( Tel ) 　　　 （ 　　　　 ） ", + "9": "有効期間", + "11": "　　年 月 日 \n 　～　　年 月 日 " + }, + "links": null + }, + { + "r": 37, + "c": { + "1": "業 種 　\nType of industry", + "3": "１ 販売業(Sales) ２ 整備業( Maintenance Services) \n３ 個人(Personal) ", + "9": "返納月日 ", + "11": "年 月 日 " + }, + "links": null + }, + { + "r": 39, + "c": { + "1": "番号標受領者氏名・住所\nRecipient name\nApplicant's Address", + "3": "※申請人と異なる場合のみ記入　", + "9": "備 考 ", + "17": "受付" + }, + "links": null + }, + { + "r": 42, + "c": { + "9": "返納期限 年　　　月　　　日まで" + }, + "links": null + } + ], + "裏": [ + { + "r": 1, + "c": { + "0": "◎ 注意事項" + }, + "links": null + }, + { + "r": 3, + "c": { + "0": "1 不正に許可を受けた場合は、 １年以下の懲役若しくは50万円以下の罰金、 又はこれが併科されます。\n　　（道路運送車両法第107条）" + }, + "links": null + }, + { + "r": 5, + "c": { + "0": "２ 許可証、番号標の有効期限が満了したときは、その日から５日以内に返納してください。この返納期限内に許可証、\n　　番号標を返納しないときは、 ６箇月以下の懲役または30万円以下の罰金が科せられます。\n　　（道路運送車両法第108条）" + }, + "links": null + }, + { + "r": 7, + "c": { + "0": "３ 許可を受けた自動車であっても保安基準に適合しなければ、運行してはなりません。" + }, + "links": null + }, + { + "r": 9, + "c": { + "0": "４ 上記１～３に該当すると思われる場合は、 本申請に関する情報を管轄する警察署に情報提供します。" + }, + "links": null + }, + { + "r": 11, + "c": { + "0": "◎ 臨時運行許可を申請する方は、 下記の書類を必ず提示してください。\n１ 自動車検査証、登録識別情報等通知書、自動車検査証返納証明書、登録事項等証明書など。\n２ 自動車損害賠償責任保険証明書 （自動車損害賠償責任共済証明書を含む。） " + }, + "links": null + }, + { + "r": 13, + "c": { + "0": "３ 申請人又は来庁者の住所が確認できるもの。\n 自動車運転免許証、マイナンバーカード、在留カードなど。" + }, + "links": null + }, + { + "r": 15, + "c": { + "0": "◎ 申請書記載方法\n１ 車名は、トヨタ、ニッサン、ホンダ、マツダ 等と記入してください。\n２ 形状は、該当番号に○印をつけてください。「６その他」の場合は、（ ）内に自動車検査証上の車体の形状を記入してください。\n３ 車台番号は、車台に打刻されている記号番号を記入してください。\n４ 運行の目的は、該当番号に一つだけ○印をつけてください。「３その他」の場合は、（ ）内に具体的に記入してください。" + }, + "links": null + }, + { + "r": 17, + "c": { + "0": "５ 運行の経路は、運行目的達成のための発着主要経路の地点名を記入してください。\n　 　（例 千代田区霞ヶ関～◯◯市～◯◯高速～◯◯市◯◯区）\n　　 したがって、都道府県内一円、市、町内等ばく然とした地域を記入したもの、車検切れの車を販売する等の目的で各地を\n　　巡回する場合等は許可できません。" + }, + "links": null + }, + { + "r": 18, + "c": { + "0": "６ 許可を受ける方は、申請人欄に必ず記入（申請人と来庁者が異なる場合は番号標受領者欄も記入）してください。" + }, + "links": null + } + ], + "表 (記入例)": [ + { + "r": 1, + "c": { + "0": "様式第１号（第２条関係）" + }, + "links": null + }, + { + "r": 2, + "c": { + "5": "自動車臨時運行許可申請書" + }, + "links": null + }, + { + "r": 3, + "c": { + "5": "APPLICATION FOR CAR TEMPORARY PLATE" + }, + "links": null + }, + { + "r": 4, + "c": { + "2": "※注：裏面をよく読んで太線内を記入し、 必要な書類を添えて提出してください。" + }, + "links": null + }, + { + "r": 5, + "c": { + "0": "車名\nMaker of the vehicle", + "2": "トヨタ　などの車名" + }, + "links": null + }, + { + "r": 7, + "c": { + "0": "形状\nType of Body", + "2": "１　箱形（Box-shaped） ２　ステーションワゴン（Station Wagon） " + }, + "links": null + }, + { + "r": 8, + "c": { + "2": "３　バン（Van） ４　キャブオーバー（Cab-over） " + }, + "links": null + }, + { + "r": 9, + "c": { + "2": "５　オートバイ （ motorcycle ） ６　その他( 　　　　　　　　　 ) " + }, + "links": null + }, + { + "r": 10, + "c": { + "0": "車台番号\nSerial No. ", + "2": "ＡＢＣ－Ａ１２３４５", + "9": "自動車損害賠償責任保険　Car Insurance " + }, + "links": null + }, + { + "r": 11, + "c": { + "0": "運行の目的\nPurpose", + "2": "１　車検のための回送( Inspection )　 ２　登録のための回送( Registration ) ", + "9": "保険会社名 \nName of Co.", + "11": "　　　　○○○○　　保険会社 " + }, + "links": null + }, + { + "r": 12, + "c": { + "2": "３　封印取付け( Seal )のための回送 " + }, + "links": null + }, + { + "r": 13, + "c": { + "9": "証明書番号\nVoucher No.", + "11": "ＡＢＣ1234ＣＢＡ" + }, + "links": null + }, + { + "r": 14, + "c": { + "2": "４　その他 （other） （ ） " + }, + "links": null + }, + { + "r": 15, + "c": { + "0": "運行の経路\nRoute ", + "2": "※発着主要経路の地点名を記入してください。\n出発地（From）　　　　　　　　　経由地（Via）　　　　　　　　　到着地（To） \n　　　　　　　　～　　　　　　　　　　　　　　　　　　～", + "9": "保険期間\nInsurannce\nPeriod ", + "11": "自(From) ", + "15": "年", + "16": "月", + "17": "日" + }, + "links": null + }, + { + "r": 16, + "c": { + "11": "至(To) ", + "15": "年", + "16": "月", + "17": "日" + }, + "links": null + }, + { + "r": 17, + "c": { + "9": "備 考 " + }, + "links": null + }, + { + "r": 18, + "c": { + "0": "運行の期間\nService period", + "2": "自(From) 年 月 日 ～ \n至(To) 年 月 日 （ 日間） " + }, + "links": null + }, + { + "r": 19, + "c": { + "2": "※目的達成に必要な最小限の日数を記入してください。 " + }, + "links": null + }, + { + "r": 20, + "c": { + "2": "（通常、整備のための回送は１日間、車検・登録のための回送は、１～２日間です。) " + }, + "links": null + }, + { + "r": 21, + "c": { + "1": "裏面の注意事項に同意の上、上記のとおり臨時運行の許可を申請します。" + }, + "links": null + }, + { + "r": 22, + "c": { + "12": "年 月 日" + }, + "links": null + }, + { + "r": 24, + "c": { + "4": "湖南市長宛" + }, + "links": null + }, + { + "r": 25, + "c": { + "9": "番号標番号 ", + "11": " 枚数 " + }, + "links": null + }, + { + "r": 26, + "c": { + "0": "申 請 人", + "1": "住所\nApplicant's Address", + "3": "湖南市中央一丁目１番地" + }, + "links": null + }, + { + "r": 27, + "c": { + "11": "滋　　 　 　― 　　　 １・２ " + }, + "links": null + }, + { + "r": 28, + "c": { + "1": "氏名又は名称\nName\n※法人の場合は\n代表者名も\n記入してください。", + "3": "個人の場合は氏名、法人の場合は法人名", + "9": "許可番号 ", + "11": "№ " + }, + "links": null + }, + { + "r": 31, + "c": { + "3": "（代表者） ", + "9": "許可年月日", + "11": "年 月 日 " + }, + "links": null + }, + { + "r": 33, + "c": { + "3": "代表取締役　湖南　太郎" + }, + "links": null + }, + { + "r": 35, + "c": { + "3": "電話( Tel ) 　　0748 （ 　71 ）×××× ", + "9": "有効期間", + "11": "　　年 月 日 \n 　～　　年 月 日 " + }, + "links": null + }, + { + "r": 37, + "c": { + "1": "業 種 　\nType of industry", + "3": "１ 販売業(Sales) ２ 整備業( Maintenance Services) \n３ 個人(Personal) ", + "9": "返納月日 ", + "11": "年 月 日 " + }, + "links": null + }, + { + "r": 39, + "c": { + "1": "番号標受領者氏名・住所\nRecipient name\nApplicant's Address", + "3": "※申請人と異なる場合のみ記入　", + "9": "備 考 ", + "17": "受付" + }, + "links": null + }, + { + "r": 42, + "c": { + "9": "返納期限 年　　　月　　　日まで" + }, + "links": null + } + ] + } +} diff --git a/tests/core/test_backend_import_isolation.py b/tests/core/test_backend_import_isolation.py new file mode 100644 index 00000000..ade64db5 --- /dev/null +++ b/tests/core/test_backend_import_isolation.py @@ -0,0 +1,218 @@ +"""Fresh-process import boundaries exercised by real extraction, not mocks.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +from typing import cast + +from openpyxl import Workbook +from openpyxl.chart import BarChart, Reference +from openpyxl.styles import Border, Side +import pytest + +_SHAPE_SAMPLE = ( + Path(__file__).resolve().parents[2] / "sample/flowchart/sample-shape-connector.xlsx" +) + + +def _probe(code: str) -> dict[str, object]: + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=60 + ) + assert result.returncode == 0, result.stderr or result.stdout + return cast(dict[str, object], json.loads(result.stdout)) + + +@pytest.mark.parametrize("suffix", [".xlsx", ".xlsm"]) +@pytest.mark.parametrize("without_pillow", [False, True]) +def test_real_light_extraction_isolates_backends( + tmp_path: Path, suffix: str, without_pillow: bool +) -> None: + path = tmp_path / f"rich{suffix}" + book = Workbook() + sheet = book.active + assert sheet is not None + sheet.title = "Data" + for row in [("Name", "Value"), ("A", 12), ("B", 24)]: + sheet.append(row) + border = Border(*(Side(style="thin") for _ in range(4))) + for row in sheet: + for cell in row: + cell.border = border + sheet.print_area = "A1:B3" + chart = BarChart() + chart.add_data( + Reference(sheet, min_col=2, min_row=1, max_row=3), titles_from_data=True + ) + sheet.add_chart(chart, "D1") + book.save(path) + book.close() + # This finder simulates an absent optional dependency only inside this probe. + # Production extraction does not modify module caches or import machinery. + payload = _probe( + f""" +import importlib.abc +import json +import os +import sys + +if {without_pillow!r}: + class MissingPillow(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "PIL" or fullname.startswith("PIL."): + raise ModuleNotFoundError("Pillow unavailable", name=fullname) + sys.meta_path.insert(0, MissingPillow()) + +os.environ["EXSTRUCT_BORDER_CLUSTER_BACKEND"] = "python" +import exstruct +workbook = exstruct.extract({str(path)!r}, mode="light") +sheet = workbook.sheets["Data"] +assert sheet.rows[1].c == {{"0": "A", "1": 12}} +assert sheet.table_candidates +assert sheet.print_areas +assert sheet.charts and sheet.charts[0].provenance == "python_ooxml" +# Existing real OOXML shapes/connectors must survive the import split too. +diagram = exstruct.extract({str(_SHAPE_SAMPLE)!r}, mode="light") +assert any(s.shapes for s in diagram.sheets.values()) +for s in diagram.sheets.values(): + assert all(shape.provenance == "python_ooxml" for shape in s.shapes) +forbidden = ["xlwings", "scipy", "pandas", "xlrd", "exstruct.core.backends.com_backend", + "exstruct.core.backends.libreoffice_backend", "exstruct.core.charts", + "exstruct.core.shapes", "exstruct.render", "pypdfium2"] +print(json.dumps({{ + "loaded": [name for name in forbidden if any( + m == name or m.startswith(name + ".") for m in sys.modules)], + "pillow": "PIL" in sys.modules, +}})) +""" + ) + assert payload["loaded"] == [] + if without_pillow: + assert payload["pillow"] is False + + +def test_shared_backend_contracts_do_not_load_concrete_dependencies() -> None: + payload = _probe(""" +import json +import sys +from exstruct.core.backends.base import Backend, RichBackend +import typing +assert typing.get_type_hints(Backend.extract_merged_cells) +assert typing.get_type_hints(RichBackend.extract_shapes) +roots = ["numpy", "pandas", "openpyxl", "xlwings", "scipy", "PIL"] +print(json.dumps({"loaded": [name for name in roots if name in sys.modules]})) +""") + assert payload["loaded"] == [] + + +def test_runtime_annotations_remain_resolvable() -> None: + payload = _probe(""" +import json +import typing +import exstruct +from exstruct.core import cells, integrate, pipeline, workbook +from exstruct.core.backends import Backend, OpenpyxlBackend +assert cells.MergedCellRange is pipeline.MergedCellRange +for function in (exstruct.extract, integrate.extract_workbook, + pipeline.resolve_rich_backend, pipeline.step_extract_shapes_com, + workbook.xlwings_workbook, cells.detect_tables, + OpenpyxlBackend.extract_cells, Backend.extract_cells): + assert typing.get_type_hints(function) +assert typing.get_type_hints(pipeline.PipelinePlan) +print(json.dumps({"resolved": True})) +""") + assert payload == {"resolved": True} + + +@pytest.mark.parametrize("backend", ["python", "numpy"]) +def test_scipy_loads_only_for_accelerated_clustering(backend: str) -> None: + payload = _probe( + f""" +import json +import os +import sys +import numpy as np +os.environ["EXSTRUCT_BORDER_CLUSTER_BACKEND"] = {backend!r} +from exstruct.core.cells import detect_border_clusters +assert "scipy" not in sys.modules +assert detect_border_clusters(np.ones((2, 2), dtype=bool)) == [(0, 0, 1, 1)] +print(json.dumps({{"scipy": "scipy" in sys.modules}})) +""" + ) + assert payload == {"scipy": backend == "numpy"} + + +def test_compatibility_wrappers_observe_live_module_overrides() -> None: + payload = _probe(""" +import json +from types import SimpleNamespace +from exstruct.core import charts, pipeline, shapes, workbook +import openpyxl +import xlwings +original_app = xlwings.App +workbook.xw.App = "live module override" +assert xlwings.App == "live module override" +workbook.xw.App = original_app +shapes.get_shapes_with_position = lambda workbook, mode: {"live": []} +charts.get_charts = lambda sheet, mode: [] +assert pipeline.get_shapes_with_position(object()) == {"live": []} +assert pipeline.get_charts(object()) == [] +openpyxl.load_workbook = lambda *args, **kwargs: "live loader" +assert workbook.load_workbook("ignored.xlsx") == "live loader" +workbook.load_workbook = lambda *args, **kwargs: SimpleNamespace(close=lambda: None) +with workbook.openpyxl_workbook("ignored.xlsx", data_only=True, read_only=False) as book: + assert hasattr(book, "close") +print(json.dumps({"live_overrides": True})) +""") + assert payload == {"live_overrides": True} + + +def test_pipeline_backend_alias_overrides_avoid_concrete_imports() -> None: + payload = _probe(""" +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock +from exstruct.core import pipeline +inputs = SimpleNamespace(mode='standard', file_path=Path('unused.xlsx'), + include_default_background=False, ignore_colors=None) +workbook = object() +com_rich = Mock(return_value='COM_OVERRIDE') +pipeline.ComRichBackend = com_rich +assert pipeline.resolve_rich_backend(inputs=inputs, workbook=workbook) == 'COM_OVERRIDE' +com_rich.assert_called_once_with(workbook) +inputs.mode = 'libreoffice' +lo_rich = Mock(return_value='LO_OVERRIDE') +pipeline.LibreOfficeRichBackend = lo_rich +assert pipeline.resolve_rich_backend(inputs=inputs) == 'LO_OVERRIDE' +lo_rich.assert_called_once_with(inputs.file_path) +backend = SimpleNamespace(extract_print_areas=Mock(return_value={'p': []}), + extract_auto_page_breaks=Mock(return_value={'a': []}), + extract_formulas_map=Mock(return_value={'f': {}}), + extract_colors_map=Mock(return_value={'c': {}})) +factory = Mock(return_value=backend) +pipeline.ComBackend = factory +artifacts = pipeline.ExtractionArtifacts() +pipeline.step_extract_print_areas_com(inputs, artifacts, workbook) +pipeline.step_extract_auto_page_breaks_com(inputs, artifacts, workbook) +pipeline.step_extract_formulas_map_com(inputs, artifacts, workbook) +pipeline.step_extract_colors_map_com(inputs, artifacts, workbook) +assert artifacts.print_area_data == {'p': []} +assert artifacts.auto_page_break_data == {'a': []} +assert artifacts.formulas_map_data == {'f': {}} +assert artifacts.colors_map_data == {'c': {}} +assert factory.call_count == 4 +for call in factory.call_args_list: + assert call.args == (workbook,) +backend.extract_colors_map.assert_called_once_with(include_default_background=False, + ignore_colors=None) +for name in ('xlwings', 'exstruct.core.backends.com_backend', + 'exstruct.core.backends.libreoffice_backend'): + assert name not in sys.modules +print(json.dumps({'overrides': True})) +""") + assert payload == {"overrides": True} diff --git a/tests/core/test_border_clustering_issue148.py b/tests/core/test_border_clustering_issue148.py new file mode 100644 index 00000000..69f4e1ba --- /dev/null +++ b/tests/core/test_border_clustering_issue148.py @@ -0,0 +1,198 @@ +"""Clustering parity and SciPy-free runtime contracts for issue 148.""" + +from importlib.metadata import PackageNotFoundError +import json +from pathlib import Path +import subprocess +import sys +from unittest.mock import Mock + +from benchmark import issue148_clustering as comparison +import numpy as np +from openpyxl import Workbook +from openpyxl.styles import Border, Side +import pytest + +import exstruct +from exstruct.core import cells + + +@pytest.mark.parametrize("min_size", [-1, 0, 1, 2, 4, 8, 100]) +def test_cluster_parity(min_size: int) -> None: + """Compare exact order and bboxes, including holes and random maps.""" + grids = [ + np.zeros((0, 4), dtype=bool), + np.zeros((4, 0), dtype=bool), + np.zeros((3, 7), dtype=bool), + np.ones((5, 8), dtype=bool), + np.eye(8, dtype=bool), + np.array([[1, 0, 1, 1], [1, 0, 0, 0], [0, 1, 1, 1]], dtype=bool), + np.array([[1, 1, 1], [1, 0, 1], [1, 1, 1]], dtype=bool), + ] + random = np.random.default_rng(148) + grids.extend(random.random((13, 19)) < density for density in (0.05, 0.3, 0.8)) + grids.extend([grids[-1].T, grids[-1][::-1, ::-1]]) + for grid in grids: + original = grid.copy() + expected = cells._detect_border_clusters_numpy(grid, min_size) + assert cells._detect_border_clusters_python(grid, min_size) == expected + assert cells._detect_border_clusters_sparse(grid, min_size) == expected + np.testing.assert_array_equal(grid, original) + + +def test_component_size_and_row_major_seed_order() -> None: + """Four neighbors, occupied-cell threshold and seed order are independent.""" + grid = np.array([[0, 0, 1, 1], [1, 0, 0, 0], [1, 1, 1, 0]], dtype=bool) + # Second seed's bbox starts left of the first: do not sort boxes lexically. + for implementation in ( + cells._detect_border_clusters_numpy, + cells._detect_border_clusters_python, + cells._detect_border_clusters_sparse, + ): + assert implementation(grid, 2) == [(0, 2, 0, 3), (1, 0, 2, 2)] + assert implementation(grid, 4) == [(1, 0, 2, 2)] + assert implementation(np.eye(4, dtype=bool), 2) == [] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("", "auto"), + ("auto", "auto"), + (" PYTHON ", "python"), + ("NumPy", "numpy"), + ("scipy", "auto"), + ("invalid", "auto"), + ], +) +def test_backend_names_unchanged( + monkeypatch: pytest.MonkeyPatch, value: str, expected: str +) -> None: + monkeypatch.setenv("EXSTRUCT_BORDER_CLUSTER_BACKEND", value) + assert cells._resolve_border_cluster_backend() == expected + + +@pytest.mark.parametrize("backend", ["auto", "numpy"]) +@pytest.mark.parametrize("error", [ImportError("blocked scipy"), RuntimeError("label")]) +def test_accelerated_failure_and_live_overrides( + monkeypatch: pytest.MonkeyPatch, backend: str, error: Exception +) -> None: + grid = np.ones((2, 2), dtype=bool) + accelerated = Mock(side_effect=error) + fallback = Mock(return_value=[(9, 8, 7, 6)]) + monkeypatch.setenv("EXSTRUCT_BORDER_CLUSTER_BACKEND", backend) + monkeypatch.setattr(cells, "_detect_border_clusters_numpy", accelerated) + monkeypatch.setattr(cells, "_detect_border_clusters_python", fallback) + assert cells.detect_border_clusters(grid, 3) == [(9, 8, 7, 6)] + accelerated.assert_called_once_with(grid, 3) + fallback.assert_called_once_with(grid, 3) + + +def test_explicit_python_skips_accelerated_backend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EXSTRUCT_BORDER_CLUSTER_BACKEND", "python") + accelerated = Mock(side_effect=AssertionError("must not call scipy")) + monkeypatch.setattr(cells, "_detect_border_clusters_numpy", accelerated) + assert cells.detect_border_clusters(np.ones((2, 2), dtype=bool)) == [(0, 0, 1, 1)] + accelerated.assert_not_called() + + +@pytest.mark.parametrize("backend", ["python", "auto", "numpy"]) +def test_light_extraction_with_scipy_imports_blocked( + tmp_path: Path, backend: str +) -> None: + """Block SciPy before importing ExStruct in a fresh child interpreter.""" + workbook = Workbook() + sheet = workbook.active + assert sheet is not None + border = Border(bottom=Side(style="thin")) + for row in range(1, 5): + for col in range(1, 4): + sheet.cell(row, col, f"value-{row}-{col}").border = border + path = tmp_path / "bordered.xlsx" + workbook.save(path) + workbook.close() + script = """ +import importlib.abc +import os +import sys +class NoScipy(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'scipy' or fullname.startswith('scipy.'): + raise ModuleNotFoundError('SciPy blocked for issue 148') +sys.meta_path.insert(0, NoScipy()) +os.environ['EXSTRUCT_BORDER_CLUSTER_BACKEND'] = sys.argv[2] +import exstruct +from exstruct.core import cells +import numpy as np +assert cells.detect_border_clusters(np.ones((2, 2), dtype=bool)) == [(0, 0, 1, 1)] +result = exstruct.extract(sys.argv[1], mode='light') +assert result.sheets['Sheet'].table_candidates +assert result.sheets['Sheet'].rows +assert not any(name == 'scipy' or name.startswith('scipy.') for name in sys.modules) +""" + child = subprocess.run( + [sys.executable, "-c", script, str(path), backend], + capture_output=True, + text=True, + timeout=90, + check=False, + ) + assert child.returncode == 0, child.stdout + child.stderr + + +def test_comparison_optional_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + """A missing optional pandas installation must not block comparison.""" + monkeypatch.setattr(comparison, "version", Mock(side_effect=PackageNotFoundError)) + assert comparison.dependency_version("pandas") is None + monkeypatch.setattr(comparison, "version", Mock(return_value="1.2.3")) + assert comparison.dependency_version("numpy") == "1.2.3" + + +@pytest.mark.parametrize( + "error", [FileNotFoundError("git"), subprocess.TimeoutExpired("git", 10)] +) +def test_comparison_git_metadata_unavailable( + monkeypatch: pytest.MonkeyPatch, error: Exception +) -> None: + monkeypatch.setattr(subprocess, "run", Mock(side_effect=error)) + assert comparison.git_metadata(["rev-parse", "HEAD"]) is None + + +def test_comparison_check_only_never_times(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(comparison, "perf_counter", Mock(side_effect=AssertionError)) + assert comparison.observations(lambda: "result", 7, True) == ("result", {}) + + +def test_comparison_main_preserves_unknown_metadata( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + output = tmp_path / "comparison.json" + monkeypatch.setattr( + sys, "argv", ["comparison", "--check-only", "--output", str(output)] + ) + monkeypatch.setattr(comparison, "git_metadata", Mock(return_value=None)) + monkeypatch.setattr(comparison, "dependency_version", Mock(return_value=None)) + monkeypatch.setattr(comparison, "run_comparison", Mock(return_value={})) + assert comparison.main() == 0 + result = json.loads(output.read_text(encoding="utf-8")) + assert result["source_revision"] is None + assert result["source_dirty"] is None + assert result["dependencies"]["pandas"] is None + assert result["warm_repeats"] == 0 + + +def test_comparison_extraction_restores_live_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The evaluator restores monkeypatch/environment surfaces on failure.""" + original = cells._detect_border_clusters_python + monkeypatch.setenv("EXSTRUCT_BORDER_CLUSTER_BACKEND", "auto") + monkeypatch.setattr(exstruct, "extract", Mock(side_effect=ValueError)) + with pytest.raises(ValueError): + comparison.extraction( + Path("not-used.xlsx"), cells._detect_border_clusters_sparse + ) + assert cells._detect_border_clusters_python is original + assert cells._resolve_border_cluster_backend() == "auto" diff --git a/tests/core/test_cells_reader_parity.py b/tests/core/test_cells_reader_parity.py new file mode 100644 index 00000000..197d8859 --- /dev/null +++ b/tests/core/test_cells_reader_parity.py @@ -0,0 +1,303 @@ +"""Cell contracts captured against the pandas reader before its replacement.""" + +from datetime import date, datetime, time, timedelta +import json +from pathlib import Path +import re +import subprocess +import sys +from unittest.mock import MagicMock +from zipfile import ZIP_DEFLATED, ZipFile + +from openpyxl import Workbook +from openpyxl.worksheet.hyperlink import Hyperlink +import pytest +import xlrd + +from exstruct.core import cells +from exstruct.models import CellRow + + +def make_parity_book(path: Path) -> None: + """Cover reader conversions, missing tokens, cached formulas and row gaps.""" + wb = Workbook() + ws = wb.active + ws.title = "Values" + ws.append([123, 1.5, "00123", "1.50", True, False, " ", " x "]) + ws.append([0, -5, 1e20, "1e2", "+001", "-.50", "1.", " NA "]) + ws.append( + [ + date(2020, 1, 2), + datetime(2020, 1, 2, 3, 4, 5, 123000), + time(3, 4, 5, 123000), + timedelta(days=2, hours=3), + ] + ) + tokens = [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "", + "N/A", + "NA", + "NULL", + "NaN", + "None", + "n/a", + "nan", + "null", + ] + for col, token in enumerate(tokens, start=1): + ws.cell(4, col, token).data_type = "s" + for col, error in enumerate( + ["#DIV/0!", "#VALUE!", "#REF!", "#NAME?", "#NUM!", "#NULL!"], start=1 + ): + ws.cell(5, col, error) + ws["D9"] = "sparse" + ws["A10"] = "=1+1" + ws["B10"] = "=TRUE()" + ws["C10"] = '= "cached"' + ws["D10"] = "=1/0" + ws["E10"] = "=1+2" + ws["A11"] = "visible" + ws["B11"].hyperlink = "https://example.test/na" + ws["B11"] = "NA" + ws["A12"].hyperlink = "https://example.test/only" + ws["A12"] = "NA" + ws["C11"].hyperlink = Hyperlink(ref="C11", location="Values!A1") + ws["C11"] = "internal" + ws["A14"] = "first" + ws.merge_cells("A14:B14") + wb.create_sheet("Empty") + wb.create_sheet("Last")["C3"] = "last" + wb.save(path) + wb.close() + with ZipFile(path) as archive: + parts = {name: archive.read(name) for name in archive.namelist()} + sheet = parts["xl/worksheets/sheet1.xml"].decode().replace("", "") + sheet = sheet.replace("1+1", "1+12") + sheet = sheet.replace("TRUE()", "TRUE()1") + sheet = sheet.replace('r="B10"', 'r="B10" t="b"') + sheet = sheet.replace('= "cached"', '= "cached"cached') + # The formula text is stored without its initial equals sign. + sheet = sheet.replace(' "cached"', ' "cached"cached') + sheet = sheet.replace('r="C10"', 'r="C10" t="str"') + sheet = sheet.replace("1/0", "1/0#DIV/0!") + sheet = sheet.replace('r="D10"', 'r="D10" t="e"') + parts["xl/worksheets/sheet1.xml"] = sheet.encode() + with ZipFile(path, "w", ZIP_DEFLATED) as archive: + for name, content in parts.items(): + archive.writestr(name, content) + + +def dump_rows(data: dict[str, list[CellRow]]) -> dict[str, list[dict[str, object]]]: + return { + name: [row.model_dump(mode="json") for row in rows] + for name, rows in data.items() + } + + +@pytest.mark.parametrize("suffix", [".xlsx", ".xlsm"]) +def test_cells_match_frozen_old_reader(tmp_path: Path, suffix: str) -> None: + path = tmp_path / f"parity{suffix}" + make_parity_book(path) + expected = json.loads( + (Path(__file__).parent / "cells_reader_expected.json").read_text( + encoding="utf-8" + ) + ) + assert dump_rows(cells.extract_sheet_cells(path)) == expected["xlsx"] + assert dump_rows(cells.extract_sheet_cells_with_links(path)) == expected["links"] + + +def test_real_xls_matches_frozen_old_reader() -> None: + path = Path(__file__).parents[1] / "assets" / "sample.xls" + expected = json.loads( + (Path(__file__).parent / "cells_reader_expected.json").read_text( + encoding="utf-8" + ) + ) + assert dump_rows(cells.extract_sheet_cells(path)) == expected["xls"] + + +def test_cells_match_live_pandas_oracle(tmp_path: Path) -> None: + pd = pytest.importorskip("pandas") + path = tmp_path / "oracle.xlsx" + make_parity_book(path) + expected: dict[str, list[CellRow]] = {} + for name, frame in pd.read_excel( + path, header=None, sheet_name=None, dtype=str + ).items(): + rows = [] + for row_number, row in enumerate( + frame.fillna("").itertuples(index=False, name=None), start=1 + ): + values = { + str(col): cells._coerce_numeric_preserve_format(str(value)) + for col, value in enumerate(row) + if str(value).strip() + } + if values: + rows.append(CellRow(r=row_number, c=values)) + expected[name] = rows + assert cells.extract_sheet_cells(path) == expected + + +@pytest.mark.parametrize("datemode", [0, 1]) +def test_xls_typed_cached_values_match_old_oracle( + monkeypatch: pytest.MonkeyPatch, datemode: int +) -> None: + """BIFF cell types include cached formulas; conversion ignores formula text.""" + pd = pytest.importorskip("pandas") + types = [xlrd.XL_CELL_DATE] * 4 + [ + xlrd.XL_CELL_BOOLEAN, + xlrd.XL_CELL_BOOLEAN, + xlrd.XL_CELL_ERROR, + xlrd.XL_CELL_NUMBER, + xlrd.XL_CELL_NUMBER, + xlrd.XL_CELL_NUMBER, + xlrd.XL_CELL_TEXT, + xlrd.XL_CELL_TEXT, + xlrd.XL_CELL_TEXT, + xlrd.XL_CELL_NUMBER, + xlrd.XL_CELL_NUMBER, + ] + values = [ + 0.5, + 1.0, + 59.0, + 60.0, + 1, + 0, + 7, + 2.0, + 1.25, + 1e20, + "001", + "NA", + " NA ", + float("inf"), + float("nan"), + ] + row_cells = [ + xlrd.sheet.Cell(typ, value) for typ, value in zip(types, values, strict=True) + ] + sheet = MagicMock() + sheet.name = "Cached" + sheet.nrows = 3 + blank_cells = [xlrd.sheet.Cell(xlrd.XL_CELL_EMPTY, "") for _ in values] + sheet.row.side_effect = lambda index: row_cells if index == 2 else blank_cells + sheet.row_values.side_effect = lambda index: ( + values if index == 2 else [""] * len(values) + ) + sheet.row_types.side_effect = lambda index: ( + types if index == 2 else [xlrd.XL_CELL_EMPTY] * len(values) + ) + book = MagicMock(spec=xlrd.Book) + book.datemode = datemode + book.sheets.return_value = [sheet] + book.sheet_names.return_value = ["Cached"] + book.sheet_by_name.return_value = sheet + monkeypatch.setattr(xlrd, "open_workbook", lambda *args, **kwargs: book) + frame = pd.read_excel( + book, engine="xlrd", sheet_name="Cached", header=None, dtype=str + ).fillna("") + expected: dict[str, int | float | str] = { + str(column): cells._coerce_numeric_preserve_format(str(value)) + for column, value in enumerate(frame.iloc[2]) + if str(value).strip() + } + actual = cells.extract_sheet_cells(Path("cached.XLS")) + assert actual == {"Cached": [CellRow(r=3, c=expected)]} + assert actual["Cached"][0].c["0"] == "12:00:00" + assert actual["Cached"][0].c["4"] == "True" + assert actual["Cached"][0].c["5"] == "False" + assert "6" not in actual["Cached"][0].c + assert "11" not in actual["Cached"][0].c + assert actual["Cached"][0].c["12"] == " NA " + book.release_resources.assert_called() + + +def test_xls_release_on_read_failure(monkeypatch: pytest.MonkeyPatch) -> None: + book = MagicMock(spec=xlrd.Book) + book.sheets.side_effect = RuntimeError("broken sheet") + monkeypatch.setattr(xlrd, "open_workbook", lambda *args, **kwargs: book) + with pytest.raises(RuntimeError, match="broken sheet"): + cells.extract_sheet_cells(Path("broken.xls")) + book.release_resources.assert_called_once() + + +def test_xls_date_overflow_keeps_old_serial() -> None: + assert cells._xls_cell_value(1e20, xlrd.XL_CELL_DATE, 0) == 1e20 + + +def test_ws_helper_matches_path_and_keeps_workbook_open(tmp_path: Path) -> None: + path = tmp_path / "helper.xlsx" + make_parity_book(path) + with cells.openpyxl_workbook(path, data_only=True, read_only=False) as wb: + rows = cells.extract_sheet_cells_openpyxl_ws(wb["Values"], include_links=True) + assert rows == cells.extract_sheet_cells_with_links(path)["Values"] + assert wb["Values"]["A10"].value == 2 + assert wb["Values"]["C10"].value == "cached" + assert not any(row.r == 12 for row in rows) + assert next(row for row in rows if row.r == 11).links == { + "1": "https://example.test/na" + } + + +def test_xlsx_works_without_pandas_and_keeps_xlrd_lazy(tmp_path: Path) -> None: + path = tmp_path / "imports.xlsx" + make_parity_book(path) + code = ( + "import sys; sys.modules['pandas'] = None; from pathlib import Path; " + "from exstruct.core.cells import extract_sheet_cells; " + "extract_sheet_cells(Path(sys.argv[1])); " + "assert sys.modules['pandas'] is None; assert 'xlrd' not in sys.modules" + ) + subprocess.run([sys.executable, "-c", code, str(path)], check=True) + + +def test_streaming_reader_ignores_incorrect_declared_dimensions(tmp_path: Path) -> None: + """The former pandas/openpyxl reader reset bounds before reading rows.""" + path = tmp_path / "dimensions.xlsx" + wb = Workbook() + wb.active["C3"] = "first" + wb.active["D9"] = "last" + wb.save(path) + wb.close() + with ZipFile(path) as archive: + parts = {name: archive.read(name) for name in archive.namelist()} + parts["xl/worksheets/sheet1.xml"] = re.sub( + rb' None: + workbook = Workbook() + sheet = workbook.active + assert sheet is not None + sheet["C3"] = "first" + sheet["AX1000"] = "last" + sheet["AX1000"].hyperlink = "https://example.test/last" + before = set(sheet._cells) + assert cells.extract_sheet_cells_openpyxl_ws(sheet, include_links=True) == [ + CellRow(r=3, c={"2": "first"}), + CellRow(r=1000, c={"49": "last"}, links={"49": "https://example.test/last"}), + ] + assert set(sheet._cells) == before + workbook.close() diff --git a/tests/core/test_openpyxl_extraction_session.py b/tests/core/test_openpyxl_extraction_session.py new file mode 100644 index 00000000..c501fdc4 --- /dev/null +++ b/tests/core/test_openpyxl_extraction_session.py @@ -0,0 +1,412 @@ +"""Extraction resource sharing, isolation and compatibility contracts (#146).""" + +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path +from threading import Barrier +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import Mock + +from openpyxl import Workbook +from openpyxl.styles import Border, PatternFill, Side +from openpyxl.worksheet.table import Table +import pytest + +from exstruct.core import cells, pipeline, workbook +from exstruct.core.backends import openpyxl_backend +from exstruct.core.backends.openpyxl_backend import OpenpyxlBackend +from exstruct.core.integrate import extract_workbook +from exstruct.core.openpyxl_session import OpenpyxlExtractionSession +from exstruct.core.pipeline import ExtractionInputs, ExtractionMode + + +def make_workbook(path: Path, sheets: int = 3) -> None: + wb = Workbook() + wb.remove(wb.active) + for index in range(sheets): + ws = wb.create_sheet(f"Sheet{index}") + ws.append(["Name", "Count"]) + ws.append(["Item", 7]) + ws["A2"].hyperlink = "https://example.com" + ws["B2"].fill = PatternFill("solid", fgColor="FF123456") + ws["D1"] = "merged" + ws.merge_cells("D1:E1") + ws["F1"] = "=SUM(B2)" + ws.print_area = "A1:F3" + ws.add_table(Table(displayName=f"Table{index}", ref="A1:B2")) + edge = Side(style="thin") + for row in ws.iter_rows(min_row=1, max_row=2, min_col=1, max_col=2): + for cell in row: + cell.border = Border(top=edge, bottom=edge, left=edge, right=edge) + wb.save(path) + wb.close() + + +def inputs_for(path: Path, *, formulas: bool = False) -> ExtractionInputs: + return pipeline.resolve_extraction_inputs( + path, + mode="light", + include_cell_links=True, + include_print_areas=True, + include_auto_page_breaks=False, + include_colors_map=True, + include_default_background=False, + ignore_colors=set(), + include_formulas_map=formulas, + include_merged_cells=True, + include_merged_values_in_rows=False, + ) + + +def track_loads(monkeypatch: pytest.MonkeyPatch) -> tuple[list[bool], list[Mock]]: + original = workbook.load_workbook + variants: list[bool] = [] + closes: list[Mock] = [] + + def load(path: Path, *, data_only: bool, read_only: bool) -> Any: # noqa: ANN401 + assert read_only is False + variants.append(data_only) + wb = original(path, data_only=data_only, read_only=read_only) + close = Mock(wraps=wb.close) + wb.close = close + closes.append(close) + return wb + + monkeypatch.setattr(workbook, "load_workbook", load) + return variants, closes + + +@pytest.mark.parametrize("suffix", [".xlsx", ".xlsm"]) +@pytest.mark.parametrize("formulas", [False, True]) +@pytest.mark.parametrize("sheet_count", [1, 4]) +def test_pipeline_loads_once_per_variant_and_closes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + suffix: str, + formulas: bool, + sheet_count: int, +) -> None: + path = tmp_path / f"book{suffix}" + make_workbook(path, sheet_count) + variants, closes = track_loads(monkeypatch) + result = pipeline.run_extraction_pipeline(inputs_for(path, formulas=formulas)) + assert variants == ([True, False] if formulas else [True]) + assert len(result.workbook.sheets) == sheet_count + for sheet in result.workbook.sheets.values(): + assert "A1:B2" in sheet.table_candidates + assert sheet.print_areas + assert sheet.merged_cells + assert sheet.colors_map + assert bool(sheet.formulas_map) is formulas + assert all(close.call_count == 1 for close in closes) + + +def test_session_is_lazy_closed_and_rejects_reuse( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + variants, _ = track_loads(monkeypatch) + session = OpenpyxlExtractionSession(tmp_path / "absent.xlsx") + with session: + assert variants == [] + with pytest.raises(RuntimeError, match="closed"): + session.workbook() + with pytest.raises(RuntimeError, match="closed"): + session.__enter__() + + +@pytest.mark.parametrize("same_file", [False, True]) +def test_com_table_detection_reuses_only_matching_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, same_file: bool +) -> None: + """Equal sheet names must not select tables from a different workbook.""" + session_path = tmp_path / "session.xlsx" + make_workbook(session_path, 1) + target_path = tmp_path / "target.xlsx" + wb = Workbook() + ws = wb.active + assert ws is not None + ws.title = "Sheet0" + ws.append(["Name", "Count", "Other"]) + ws.append(["Item", 7, 9]) + ws.add_table(Table(displayName="Target", ref="A1:C2")) + wb.save(target_path) + wb.close() + monkeypatch.chdir(tmp_path) + sheet_path = session_path if same_file else target_path + sheet = SimpleNamespace( + name="Sheet0", book=SimpleNamespace(fullname=sheet_path.name) + ) + expected = cells.detect_tables_openpyxl(sheet_path, sheet.name, mode="light") + with OpenpyxlExtractionSession(session_path) as session: + shared = session.workbook() + loader = Mock(side_effect=AssertionError("session must already be loaded")) + monkeypatch.setattr(session, "workbook", loader) + if same_file: + loader.side_effect = None + loader.return_value = shared + assert ( + cells.detect_tables(sheet, mode="light", openpyxl_session=session) + == expected + ) + assert loader.call_count == int(same_file) + assert expected == (["A1:B2"] if same_file else ["A1:C2"]) + + +def test_com_table_detection_preserves_helper_override_with_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "absent.xlsx" + sheet = SimpleNamespace(name="Sheet0", book=SimpleNamespace(fullname=str(path))) + helper = Mock(return_value=["D1:E2"]) + monkeypatch.setattr(cells, "detect_tables_openpyxl", helper) + with OpenpyxlExtractionSession(path) as session: + assert cells.detect_tables(sheet, mode="light", openpyxl_session=session) == [ + "D1:E2" + ] + helper.assert_called_once_with(path, "Sheet0", mode="light") + + +def test_close_all_variants_on_pipeline_model_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + _, closes = track_loads(monkeypatch) + monkeypatch.setattr( + pipeline, "build_workbook_data", Mock(side_effect=RuntimeError("model error")) + ) + with pytest.raises(RuntimeError, match="model error"): + pipeline.run_extraction_pipeline(inputs_for(path, formulas=True)) + assert len(closes) == 2 + assert all(close.call_count == 1 for close in closes) + + +def test_formula_load_failure_releases_regular_workbook( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + _, closes = track_loads(monkeypatch) + original = workbook.load_workbook + + def fail_formula(path: Path, *, data_only: bool, read_only: bool) -> Any: # noqa: ANN401 + if not data_only: + raise ValueError("formula parse error") + return original(path, data_only=data_only, read_only=read_only) + + monkeypatch.setattr(workbook, "load_workbook", fail_formula) + result = pipeline.run_extraction_pipeline(inputs_for(path, formulas=True)) + assert result.workbook.sheets["Sheet0"].formulas_map == {} + assert len(closes) == 1 and closes[0].call_count == 1 + + +def test_enter_failure_closes_successfully_opened_resources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + _, closes = track_loads(monkeypatch) + + class FailingSession(OpenpyxlExtractionSession): + def __enter__(self) -> OpenpyxlExtractionSession: + self.workbook() + raise ValueError("enter failure") + + monkeypatch.setattr(pipeline, "OpenpyxlExtractionSession", FailingSession) + with pytest.raises(ValueError, match="enter failure"): + pipeline.run_extraction_pipeline(inputs_for(path)) + assert closes[0].call_count == 1 + + +def test_standalone_helpers_match_shared_backend(tmp_path: Path) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + standalone = OpenpyxlBackend(path) + with OpenpyxlExtractionSession(path) as session: + backend = OpenpyxlBackend(path, session=session) + assert backend.extract_cells(include_links=True) == standalone.extract_cells( + include_links=True + ) + assert backend.extract_print_areas() == standalone.extract_print_areas() + assert backend.extract_merged_cells() == standalone.extract_merged_cells() + assert backend.extract_formulas_map() == standalone.extract_formulas_map() + assert backend.extract_colors_map( + include_default_background=True, ignore_colors=set() + ) == standalone.extract_colors_map( + include_default_background=True, ignore_colors=set() + ) + for name in session.workbook().sheetnames: + assert backend.detect_tables(name) == standalone.detect_tables(name) + actual = cells.load_border_maps_openpyxl_ws(session.workbook()[name]) + expected = cells.load_border_maps_xlsx(path, name) + for left, right in zip(actual[:5], expected[:5], strict=True): + assert (left == right).all() + assert actual[5:] == expected[5:] + + +def test_default_background_merged_cell_fallback_parity( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Keep the preexisting merged-cell color error and warning contract.""" + path = tmp_path / "book.xlsx" + make_workbook(path) + assert ( + OpenpyxlBackend(path).extract_colors_map( + include_default_background=True, + ignore_colors=set(), + ) + is None + ) + with OpenpyxlExtractionSession(path) as session: + assert ( + OpenpyxlBackend(path, session=session).extract_colors_map( + include_default_background=True, + ignore_colors=set(), + ) + is None + ) + assert caplog.text.count("Color map extraction failed; skipping colors_map") == 2 + + +def test_all_feature_model_matches_standalone_helpers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + inputs = inputs_for(path, formulas=True) + shared = pipeline.run_extraction_pipeline(inputs).workbook + monkeypatch.setattr(openpyxl_backend, "_use_session", lambda *_: False) + standalone = pipeline.run_extraction_pipeline(inputs).workbook + assert shared.model_dump() == standalone.model_dump() + + +def test_prestep_exception_releases_loaded_workbook( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + _, closes = track_loads(monkeypatch) + + def fail_after_load( + inputs: ExtractionInputs, artifacts: pipeline.ExtractionArtifacts + ) -> None: + assert artifacts.openpyxl_session is not None + artifacts.openpyxl_session.workbook() + raise RuntimeError("prestep failure") + + monkeypatch.setattr(pipeline, "build_pre_com_pipeline", lambda _: [fail_after_load]) + with pytest.raises(RuntimeError, match="prestep failure"): + pipeline.run_extraction_pipeline(inputs_for(path)) + assert len(closes) == 1 and closes[0].call_count == 1 + + +def test_fresh_extraction_observes_file_changes(tmp_path: Path) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path, 1) + first = extract_workbook(path, mode="light") + wb = workbook.load_workbook(path) + wb["Sheet0"]["A2"] = "Changed" + wb.save(path) + wb.close() + second = extract_workbook(path, mode="light") + assert first.sheets["Sheet0"].rows != second.sheets["Sheet0"].rows + + +def test_concurrent_extractions_have_independent_workbooks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + barrier = Barrier(2) + original = workbook.load_workbook + loaded: list[object] = [] + + def load(path: Path, **kwargs: Any) -> Any: # noqa: ANN401 + wb = original(path, **kwargs) + loaded.append(wb) + barrier.wait(timeout=10) + return wb + + monkeypatch.setattr(workbook, "load_workbook", load) + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [ + pool.submit(pipeline.run_extraction_pipeline, inputs_for(path)) + for _ in range(2) + ] + results = [future.result(timeout=20) for future in futures] + assert len(loaded) == 2 and loaded[0] is not loaded[1] + assert results[0].workbook == results[1].workbook + + +@pytest.mark.parametrize("module", [cells, openpyxl_backend]) +def test_legacy_helper_override_is_honored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, module: ModuleType +) -> None: + path = tmp_path / "absent.xlsx" + fake = Mock(return_value={"Override": []}) + monkeypatch.setattr(module, "extract_sheet_cells", fake) + with OpenpyxlExtractionSession(path) as session: + assert OpenpyxlBackend(path, session=session).extract_cells( + include_links=False + ) == {"Override": []} + fake.assert_called_once_with(path) + + +@pytest.mark.parametrize("success", [True, False]) +def test_mock_com_success_and_fallback_share_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, success: bool +) -> None: + path = tmp_path / "book.xlsx" + make_workbook(path) + variants, closes = track_loads(monkeypatch) + monkeypatch.delenv("SKIP_COM_TESTS", raising=False) + inputs = replace(inputs_for(path), mode="standard") + monkeypatch.setattr(pipeline, "build_com_pipeline", lambda _: []) + + @contextmanager + def com_context(_: Path) -> Iterator[Any]: + if not success: + raise RuntimeError("COM unavailable") + yield SimpleNamespace( + sheets={ + name: SimpleNamespace( + name=name, book=SimpleNamespace(fullname=str(path)) + ) + for name in ("Sheet0", "Sheet1", "Sheet2") + } + ) + + monkeypatch.setattr(pipeline, "xlwings_workbook", com_context) + result = pipeline.run_extraction_pipeline(inputs) + assert result.state.com_succeeded is success + assert variants == [True] + assert closes[0].call_count == 1 + assert all( + "A1:B2" in sheet.table_candidates for sheet in result.workbook.sheets.values() + ) + + +@pytest.mark.parametrize("mode", ["light", "standard", "verbose"]) +def test_public_xls_extraction_preserves_cells_and_com_formula_boundary( + monkeypatch: pytest.MonkeyPatch, + mode: ExtractionMode, +) -> None: + path = Path(__file__).resolve().parents[1] / "assets" / "sample.xls" + monkeypatch.setenv("SKIP_COM_TESTS", "1") + original = workbook.load_workbook + variants: list[bool] = [] + + def load(path: Path, *, data_only: bool, read_only: bool) -> Any: # noqa: ANN401 + variants.append(data_only) + return original(path, data_only=data_only, read_only=read_only) + + monkeypatch.setattr(workbook, "load_workbook", load) + result = extract_workbook(path, mode=mode) + assert result.sheets and any(sheet.rows for sheet in result.sheets.values()) + assert False not in variants # BIFF formulas must remain COM-only. diff --git a/uv.lock b/uv.lock index 896b22e7..5cb2c801 100644 --- a/uv.lock +++ b/uv.lock @@ -928,10 +928,8 @@ dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "openpyxl" }, - { name = "pandas" }, { name = "pydantic" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "xlrd" }, { name = "xlwings" }, ] @@ -943,6 +941,12 @@ all = [ { name = "pypdfium2" }, { name = "python-toon" }, { name = "pyyaml" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +fast = [ + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] mcp = [ { name = "httpx" }, @@ -965,6 +969,7 @@ dev = [ { name = "mkdocs-material" }, { name = "mkdocstrings-python" }, { name = "mypy" }, + { name = "pandas" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -982,7 +987,6 @@ requires-dist = [ { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.25.0,<2.0.0" }, { name = "numpy", specifier = ">=2.3.5" }, { name = "openpyxl", specifier = ">=3.1.5" }, - { name = "pandas", specifier = ">=2.3.3" }, { name = "pillow", marker = "extra == 'all'", specifier = ">=12.0.0" }, { name = "pillow", marker = "extra == 'render'", specifier = ">=12.0.0" }, { name = "pydantic", specifier = ">=2.12.5" }, @@ -992,10 +996,12 @@ requires-dist = [ { name = "python-toon", marker = "extra == 'toon'", specifier = ">=0.1.3" }, { name = "pyyaml", marker = "extra == 'all'", specifier = ">=6.0.3" }, { name = "pyyaml", marker = "extra == 'yaml'", specifier = ">=6.0.3" }, - { name = "scipy", specifier = ">=1.16.3" }, + { name = "scipy", marker = "extra == 'all'", specifier = ">=1.16.3" }, + { name = "scipy", marker = "extra == 'fast'", specifier = ">=1.16.3" }, + { name = "xlrd", specifier = ">=2.0.2" }, { name = "xlwings", specifier = ">=0.33.16" }, ] -provides-extras = ["all", "yaml", "toon", "render", "mcp"] +provides-extras = ["fast", "all", "yaml", "toon", "render", "mcp"] [package.metadata.requires-dev] dev = [ @@ -1003,6 +1009,7 @@ dev = [ { name = "mkdocs-material", specifier = ">=9.7.0" }, { name = "mkdocstrings-python", specifier = ">=2.0.1" }, { name = "mypy", specifier = ">=1.19.0" }, + { name = "pandas", specifier = ">=2.3.3" }, { name = "pre-commit", specifier = ">=4.5.0" }, { name = "pytest", specifier = ">=9.0.1" }, { name = "pytest-cov", specifier = ">=7.0.0" }, @@ -3756,6 +3763,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] +[[package]] +name = "xlrd" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, +] + [[package]] name = "xlwings" version = "0.37.4"