From d27ca6d79a7a8de59fe660c699fde7853c54bdf4 Mon Sep 17 00:00:00 2001 From: harumiWeb Date: Sat, 3 Oct 2026 18:38:14 +0900 Subject: [PATCH 1/4] feat: default light extraction to direct OOXML (#150) --- CHANGELOG.md | 12 + README.ja.md | 12 +- README.md | 12 +- .../issue150-2026-10-03-windows.json | 362 ++++++++++++++ benchmark/issue150-results.md | 48 ++ benchmark/issue150_light.py | 190 ++++++++ benchmark/performance.py | 4 +- benchmark/performance_profile.py | 33 +- ...-workbook-reuse-and-direct-cell-reading.md | 28 +- ...R-0012-optional-scipy-border-clustering.md | 10 +- ...ault-ooxml-backend-for-light-extraction.md | 100 ++++ dev-docs/adr/README.md | 1 + dev-docs/adr/decision-map.md | 5 + dev-docs/adr/index.yaml | 23 + dev-docs/adr/reviews/ADR-0013-2026-10-03.md | 77 +++ dev-docs/architecture/pipeline.md | 83 ++-- dev-docs/roadmap.md | 15 +- dev-docs/specs/excel-extraction.md | 117 +++-- dev-docs/specs/extraction-performance.md | 41 ++ dev-docs/specs/ooxml-core-extraction.md | 67 ++- dev-docs/testing/test-requirements.md | 8 +- docs/api.md | 18 +- docs/cli.md | 4 +- src/exstruct/core/backends/ooxml_backend.py | 3 +- .../core/backends/openpyxl_backend.py | 2 +- src/exstruct/core/cells.py | 56 ++- src/exstruct/core/ooxml_scalars.py | 273 +++++++++++ src/exstruct/core/ooxml_session.py | 459 ++++++++++++++++-- src/exstruct/core/pipeline.py | 154 +++++- src/exstruct/core/ranges.py | 21 +- src/exstruct/errors.py | 1 + tests/benchmark/test_performance.py | 8 +- tests/core/test_backend_import_isolation.py | 10 +- tests/core/test_light_ooxml_pipeline.py | 267 ++++++++++ tests/core/test_ooxml_merged_order_session.py | 41 ++ tests/core/test_ooxml_scalars.py | 93 ++++ .../test_ooxml_table_candidates_session.py | 103 ++++ 37 files changed, 2552 insertions(+), 209 deletions(-) create mode 100644 benchmark/baselines/issue150-2026-10-03-windows.json create mode 100644 benchmark/issue150-results.md create mode 100644 benchmark/issue150_light.py create mode 100644 dev-docs/adr/ADR-0013-default-ooxml-backend-for-light-extraction.md create mode 100644 dev-docs/adr/reviews/ADR-0013-2026-10-03.md create mode 100644 src/exstruct/core/ooxml_scalars.py create mode 100644 tests/core/test_light_ooxml_pipeline.py create mode 100644 tests/core/test_ooxml_merged_order_session.py create mode 100644 tests/core/test_ooxml_scalars.py create mode 100644 tests/core/test_ooxml_table_candidates_session.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e7ad0406..6a9ab363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project are documented in this file. This changelog ### Changed +- Switched `.xlsx` / `.xlsm` `light` extraction to the direct OOXML core and + rich pipeline using one shared ZIP. Unsupported OOXML and `colors_map` opt-in + restart the complete openpyxl compatibility pipeline with an + `ooxml_compatibility` warning; per-sheet drawing resilience is preserved. +- Kept ADR-0012's configured SciPy selection for other modes and compatibility + extraction while the direct light table heuristic selects Python BFS per + call without changing the environment variable. - Replaced production pandas cell extraction with direct openpyxl worksheet iteration and lazy xlrd reading for legacy `.xls` files, preserving existing cell normalization and serialized output. @@ -18,6 +25,11 @@ All notable changes to this project are documented in this file. This changelog ### Added +- Added light-pipeline parity, archive lifecycle and compatibility-restart + coverage, plus a comparison runner for the previous openpyxl path. The + recorded seven-input Windows benchmark and raw baseline are available in + `benchmark/issue150-results.md` and + `benchmark/baselines/issue150-2026-10-03-windows.json`. - Added an internal streaming OOXML core extraction session for cells, shared strings, formulas, hyperlinks, merged cells, defined names, print areas and explicit tables, with shared ZIP access for optional shapes and charts. diff --git a/README.ja.md b/README.ja.md index d3daf3e0..a46b0f3a 100644 --- a/README.ja.md +++ b/README.ja.md @@ -34,12 +34,12 @@ LLM/RAG 向けに検出ヒューリスティックや出力モードを調整で - **Excel → 構造化 JSON** - セル、図形、チャート、SmartArt、テーブル候補、セル結合範囲、印刷範囲/自動改ページ範囲をシート単位・範囲単位で出力。 - **出力モード** - - `light`: セル＋テーブル候補＋印刷範囲+図形・グラフ（OOXML直接解析によるbest-effort） + - `light`: `.xlsx` / `.xlsm` のセル・テーブル候補・印刷範囲・図形/グラフをOOXML直接解析で抽出（rich artifactはbest-effort） - `libreoffice`: `.xlsx/.xlsm` 向けの best-effort 非 COM モード。LibreOffice runtime があれば結合セル・図形・コネクタ・チャートを追加 - `standard`: Excel COM でテキスト付き図形＋矢印、チャート、SmartArt、セル結合範囲 - `verbose`: 全図形を幅高さ付きで出力、セルのハイパーリンクも出力。 - **数式取得** - - `formulas_map`（数式文字列 → セル座標）を openpyxl/COM で取得。`verbose` 既定、`include_formulas_map` で制御。 + - `formulas_map`（数式文字列 → セル座標）を OOXML または選択された openpyxl/COM 経路で取得。`verbose` 既定、`include_formulas_map` で制御。 - **フォーマット** - JSON（デフォルトはコンパクト、`--pretty` で整形）、YAML、TOON（任意依存）。 - **ワークブック編集インターフェース** @@ -49,7 +49,7 @@ LLM/RAG 向けに検出ヒューリスティックや出力モードを調整で - **ハイパーリンク抽出** - `verbose` モード（または `include_cell_links=True` 指定）でセルのリンクを `links` に出力。 - **安全なフォールバック** - - Excel COM または LibreOffice runtime が不在でもプロセスは落ちず、OOXML直接解析に切り替えます。 + - `.xlsx` / `.xlsm` の `light` はOOXMLを直接解析します。未対応OOXMLまたは `colors_map` 指定時はpartial結果を破棄し、openpyxl互換パイプライン全体を最初から実行して `ooxml_compatibility` warningを記録します。1シートのdrawing失敗は従来どおりそのシートだけbest-effortで処理します。 ## インストール @@ -60,7 +60,7 @@ pip install exstruct オプション依存: - YAML: `pip install pyyaml` -- 罫線クラスタリングの高速化: `pip install exstruct[fast]`（SciPy）。未導入時は既存の Python 実装へフォールバックします。NumPy は引き続き必須依存です。 +- 罫線クラスタリングの高速化: `pip install exstruct[fast]`（SciPy）。通常の直接OOXML light抽出ではPythonの罫線検出を使います。SciPyによる高速化と `EXSTRUCT_BORDER_CLUSTER_BACKEND` の選択は、他モードと互換抽出経路で利用できます。NumPy は引き続き必須依存です。 - TOON: `pip install python-toon` - レンダリング（PDF/PNG）: Excel + `pip install pypdfium2 pillow`（`mode=libreoffice` では非対応） - まとめて導入: `pip install exstruct[yaml,toon,render,fast]` @@ -276,14 +276,14 @@ set_table_detection_params( ## 出力モード -- **light**: セル＋テーブル候補＋`.xlsx` / `.xlsm` の best-effort OOXML 図形/コネクタ/チャート（COM 不要）。 +- **light**: `.xlsx` / `.xlsm` のセル・テーブル候補・印刷範囲・図形/コネクタ/チャートを直接OOXMLで抽出します。通常経路はopenpyxl、pandas、SciPy、xlwingsをimportしません。 - **standard**: テキスト付き図形＋矢印、チャート（COM ありで取得）、テーブル候補。セルのハイパーリンクは `include_cell_links=True` を指定したときのみ出力。 - **verbose**: すべての図形、チャート、`table_candidates`、ハイパーリンク、`colors_map`。 ## エラーハンドリング / フォールバック - Excel COM 不在時はセル＋テーブル候補に自動フォールバックし、`.xlsx` / `.xlsm` では利用可能な OOXML 図形/チャートも best-effort で保持します。 -- rich extraction の一部が失敗しても、ExStruct はセル＋テーブル候補を返しつつ、安全に保持できる既存の best-effort artifact は残します。 +- 1シートのdrawing失敗はそのシートの対象artifactだけを省き、健全なシートは維持します。未対応OOXMLまたは他のdirect pipeline stageの失敗時はpartial結果を破棄し、openpyxl互換pipeline全体を最初から実行して `ooxml_compatibility` を記録します。 - CLI はエラーを stdout/stderr に出し、失敗時は非ゼロ終了コード。 ## 任意レンダリング diff --git a/README.md b/README.md index 78e0f520..cf3e91b2 100644 --- a/README.md +++ b/README.md @@ -44,16 +44,16 @@ LLM/RAG pipelines and local automation. - **Excel -> structured JSON**: outputs cells, shapes, charts, SmartArt, table candidates, merged-cell ranges, print areas, and auto page-break areas by sheet or by area. - **Output modes**: - - `light`: cells + table candidates + print areas + shapes/charts (best-effort via direct OOXML parsing) + - `light`: cells + table candidates + print areas + best-effort shapes/charts via direct OOXML for `.xlsx` / `.xlsm` - `libreoffice`: best-effort non-COM mode for `.xlsx/.xlsm`. When the LibreOffice runtime is available, it adds merged cells, shapes, connectors, and charts - `standard`: Excel COM mode with texted shapes + arrows, charts, SmartArt, and merged-cell ranges - `verbose`: outputs all shapes with width/height and also emits cell hyperlinks -- **Formula extraction**: emits `formulas_map` (formula string -> cell coordinates) via openpyxl/COM. It is enabled by default in `verbose` and can be controlled with `include_formulas_map`. +- **Formula extraction**: emits `formulas_map` (formula string -> cell coordinates) via OOXML or the selected openpyxl/COM path. It is enabled by default in `verbose` and can be controlled with `include_formulas_map`. - **Formats**: JSON (compact by default, `--pretty` for formatting), YAML, and TOON (optional dependencies). - **Workbook editing interfaces**: use the editing CLI for primary ExStruct edit flows, keep MCP for host-owned safety controls, and use `exstruct.edit` only when you need the same patch contract from Python. - **Table detection tuning**: heuristics can be adjusted dynamically through the API. - **Hyperlink extraction**: in `verbose` mode, or with `include_cell_links=True`, cell links are emitted in `links`. -- **Safe fallback**: if Excel COM or the LibreOffice runtime is unavailable, the process does not crash and falls back to direct OOXML parsing. +- **Safe fallback**: `light` uses direct OOXML for `.xlsx` / `.xlsm` without Excel COM or LibreOffice. Unsupported OOXML or `colors_map` requests restart the complete openpyxl compatibility pipeline with an `ooxml_compatibility` warning; partial results are discarded. A drawing failure isolated to one sheet remains best-effort. ## Installation @@ -63,7 +63,7 @@ pip install exstruct Optional extras: -- Border-clustering acceleration: `pip install exstruct[fast]` (SciPy). Base extraction uses the existing Python fallback when SciPy is unavailable; NumPy remains a core dependency. +- Border-clustering acceleration: `pip install exstruct[fast]` (SciPy). Normal direct light extraction uses Python clustering. SciPy acceleration and `EXSTRUCT_BORDER_CLUSTER_BACKEND` selection apply to other modes and compatibility extraction; NumPy remains a core dependency. - YAML: `pip install pyyaml` - TOON: `pip install python-toon` - Rendering (PDF/PNG): Excel + `pip install pypdfium2 pillow` (`mode=libreoffice` is not supported) @@ -280,14 +280,14 @@ Higher values reduce false positives. Lower values reduce missed detections. ## Output Modes -- **light**: cells + table candidates + best-effort OOXML shapes/connectors/charts for `.xlsx` / `.xlsm` (no COM required). +- **light**: direct OOXML cells + table candidates + print areas and best-effort shapes/connectors/charts for `.xlsx` / `.xlsm`. The normal path does not import openpyxl, pandas, SciPy, or xlwings. - **standard**: texted shapes + arrows, charts (when COM is available), and table candidates. Cell hyperlinks are emitted only when `include_cell_links=True`. - **verbose**: all shapes, charts, `table_candidates`, hyperlinks, and `colors_map`. ## Error Handling / Fallback - If Excel COM is unavailable, extraction falls back to cells + table candidates automatically; `.xlsx` / `.xlsm` still preserve best-effort OOXML shapes/charts when available. -- If a rich-extraction step fails, ExStruct still returns cells + table candidates and keeps any already recovered best-effort artifacts where safe. +- A per-sheet drawing failure omits that sheet's affected drawing artifacts while preserving healthy sheets. An unsupported OOXML construct or failure in any other direct pipeline stage discards all partial OOXML data, restarts the complete openpyxl compatibility pipeline, and logs `ooxml_compatibility`. - The CLI writes errors to stdout/stderr and exits with a non-zero status on failure. ## Optional Rendering diff --git a/benchmark/baselines/issue150-2026-10-03-windows.json b/benchmark/baselines/issue150-2026-10-03-windows.json new file mode 100644 index 00000000..933fa91d --- /dev/null +++ b/benchmark/baselines/issue150-2026-10-03-windows.json @@ -0,0 +1,362 @@ +{ + "recorded_at": "2026-10-03T09:18:49.450266+00:00", + "python": "3.11.13 (main, Jul 23 2025, 00:29:09) [MSC v.1944 64 bit (AMD64)]", + "platform": "Windows-10-10.0.22631-SP0", + "source": { + "revision": "c2b5dad3dad726347717ef2ccb296bbf873bcb29", + "dirty": true + }, + "dependencies": { + "exstruct": "0.8.2", + "openpyxl": "3.1.5", + "numpy": "2.4.6", + "scipy": "1.17.1", + "defusedxml": "0.7.1", + "pydantic": "2.13.5" + }, + "results": [ + { + "input": "tasks\\issue150-fixtures\\small.xlsx", + "input_sha256": "e936689027ed6ba151c99ae74fe8d61d0f4e9cf802954008e18c622d2d8f6cee", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 399.11730005405843, + "first_extract_ms": 582.6431999448687, + "cold_in_process_ms": 983.3663001190871, + "repeated_extract_ms": [ + 11.544900014996529, + 11.858600191771984, + 10.786499828100204, + 8.785400073975325, + 8.868500124663115 + ], + "median_extract_ms": 10.786499828100204, + "peak_rss_bytes": 67543040, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "2a148d2d0a3a8410dd5eccb61d008a30f4502c566cbade403be0756dbf209234", + "worker_process_ms": 1523.8159999717027 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 418.06289996020496, + "first_extract_ms": 8.06320016272366, + "cold_in_process_ms": 428.1338001601398, + "repeated_extract_ms": [ + 5.306999897584319, + 4.777400055900216, + 4.719699965789914, + 5.392400082200766, + 5.040100077167153 + ], + "median_extract_ms": 5.040100077167153, + "peak_rss_bytes": 41529344, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "2a148d2d0a3a8410dd5eccb61d008a30f4502c566cbade403be0756dbf209234", + "worker_process_ms": 783.9408998843282 + } + }, + { + "input": "tasks\\issue150-fixtures\\large.xlsx", + "input_sha256": "72ff66fa3abde812e9ae3261530b1dd5b88c4dfd5173715b383d896b990f1b97", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 351.98510019108653, + "first_extract_ms": 1272.6747998967767, + "cold_in_process_ms": 1626.2495000846684, + "repeated_extract_ms": [ + 711.0170000232756, + 841.0445998888463, + 1083.7298999540508, + 1236.7646000348032, + 1017.5214000046253 + ], + "median_extract_ms": 1017.5214000046253, + "peak_rss_bytes": 114622464, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "dd0dcc08e28a1349e8580e2bc20942663d81b02a2236eaa1883326c1a9103b36", + "worker_process_ms": 8313.474000198767 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 415.91380001045763, + "first_extract_ms": 642.067400040105, + "cold_in_process_ms": 1060.061699943617, + "repeated_extract_ms": [ + 794.0476001240313, + 607.5293999165297, + 711.9624998886138, + 547.761800000444, + 575.2789999824017 + ], + "median_extract_ms": 607.5293999165297, + "peak_rss_bytes": 75325440, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "dd0dcc08e28a1349e8580e2bc20942663d81b02a2236eaa1883326c1a9103b36", + "worker_process_ms": 5320.234399987385 + } + }, + { + "input": "tasks\\issue150-fixtures\\many-sheet.xlsx", + "input_sha256": "608cf4728344614879790a083cf641109dfbed26dc9eafc60002fd75ae667ce0", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 328.68080004118383, + "first_extract_ms": 510.08669985458255, + "cold_in_process_ms": 839.9064000695944, + "repeated_extract_ms": [ + 157.03190001659095, + 107.7014000620693, + 94.7342999279499, + 198.85459984652698, + 91.21260000392795 + ], + "median_extract_ms": 107.7014000620693, + "peak_rss_bytes": 78163968, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "25bd3c579cdb9ae0ffdb303ad712855c5634cee89660f75dee49a96934943f8d", + "worker_process_ms": 2058.4499000106007 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 259.4261998310685, + "first_extract_ms": 72.99490016885102, + "cold_in_process_ms": 333.2481998950243, + "repeated_extract_ms": [ + 90.00999992713332, + 74.40880010835826, + 74.11119993776083, + 66.39340007677674, + 66.29049987532198 + ], + "median_extract_ms": 74.11119993776083, + "peak_rss_bytes": 44789760, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "25bd3c579cdb9ae0ffdb303ad712855c5634cee89660f75dee49a96934943f8d", + "worker_process_ms": 990.9978001378477 + } + }, + { + "input": "tasks\\issue150-fixtures\\sparse.xlsx", + "input_sha256": "39e56eda8a409f5a66df2b76b4c319e8582c5bf0440c251971ddf5c7f47c4f8e", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 294.3993997760117, + "first_extract_ms": 526.8939000088722, + "cold_in_process_ms": 823.0412998236716, + "repeated_extract_ms": [ + 46.1283999029547, + 76.15750003606081, + 43.77839993685484, + 51.29020009189844, + 125.88259996846318 + ], + "median_extract_ms": 51.29020009189844, + "peak_rss_bytes": 78983168, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "96f066e97924290a08db2c330ed9e86d2e2271536b7a7b81b16a2f822f02151b", + "worker_process_ms": 1541.1743000149727 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 232.51679982058704, + "first_extract_ms": 20.59699990786612, + "cold_in_process_ms": 253.99979995563626, + "repeated_extract_ms": [ + 21.903099957853556, + 15.564100118353963, + 16.921799862757325, + 18.670600140467286, + 16.298400005325675 + ], + "median_extract_ms": 16.921799862757325, + "peak_rss_bytes": 41627648, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "96f066e97924290a08db2c330ed9e86d2e2271536b7a7b81b16a2f822f02151b", + "worker_process_ms": 567.4101000186056 + } + }, + { + "input": "tasks\\issue150-fixtures\\style-heavy.xlsx", + "input_sha256": "05a77cc146594f2db73cb3aa764ad2625fa798809fce82fbe42900b5cc49529e", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 302.1211000159383, + "first_extract_ms": 598.1943001970649, + "cold_in_process_ms": 902.0736999809742, + "repeated_extract_ms": [ + 136.04529993608594, + 108.71589998714626, + 144.85260006040335, + 114.81840000487864, + 136.966000078246 + ], + "median_extract_ms": 136.04529993608594, + "peak_rss_bytes": 77185024, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "c2e55ebcb8c0f4705068427a831759efd9a3b8eda16593dcbc10689268255373", + "worker_process_ms": 1948.6851000692695 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 329.37669986858964, + "first_extract_ms": 129.34739980846643, + "cold_in_process_ms": 460.4908998589963, + "repeated_extract_ms": [ + 100.11130012571812, + 142.0326998922974, + 105.04289995878935, + 157.3558000382036, + 110.54809996858239 + ], + "median_extract_ms": 110.54809996858239, + "peak_rss_bytes": 48504832, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "c2e55ebcb8c0f4705068427a831759efd9a3b8eda16593dcbc10689268255373", + "worker_process_ms": 1461.7718998342752 + } + }, + { + "input": "tasks\\issue150-fixtures\\table-heavy.xlsx", + "input_sha256": "a319c15ab9b4987527beaa627ca336a679e9091f34427c08710199cfda776e5f", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 284.26939994096756, + "first_extract_ms": 537.3462000861764, + "cold_in_process_ms": 823.344900039956, + "repeated_extract_ms": [ + 86.76470001228154, + 129.96340007521212, + 79.1676000226289, + 75.20329998806119, + 136.44779985770583 + ], + "median_extract_ms": 86.76470001228154, + "peak_rss_bytes": 78946304, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "3d79c241439ca67d26b860db6c9540f10adc7676f3c395768eea611d11cfa4c7", + "worker_process_ms": 1827.428000047803 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 247.5149999372661, + "first_extract_ms": 64.0249999705702, + "cold_in_process_ms": 313.13710007816553, + "repeated_extract_ms": [ + 57.425100123509765, + 52.48690000735223, + 70.73040003888309, + 56.27709999680519, + 52.51270020380616 + ], + "median_extract_ms": 56.27709999680519, + "peak_rss_bytes": 46112768, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "3d79c241439ca67d26b860db6c9540f10adc7676f3c395768eea611d11cfa4c7", + "worker_process_ms": 973.3466999605298 + } + }, + { + "input": "sample\\flowchart\\sample-shape-connector.xlsx", + "input_sha256": "a401d6aac3a5937a2d8aecb54297e309998c5ac9f1265fde809e2cd66a0df813", + "openpyxl": { + "backend": "openpyxl", + "import_ms": 294.66040013357997, + "first_extract_ms": 546.8210000544786, + "cold_in_process_ms": 842.9002999328077, + "repeated_extract_ms": [ + 16.837300034239888, + 16.19959995150566, + 14.734199969097972, + 13.350500026717782, + 17.58919982239604 + ], + "median_extract_ms": 16.19959995150566, + "peak_rss_bytes": 67727360, + "workbook_opens": 1, + "archive_opens": 2, + "loaded": [ + "openpyxl", + "scipy" + ], + "fallback_reason": null, + "output_sha256": "1630d9cc31264fe0ab7291794b97bc47b7dc9c545f1f2b69dfa8f5eb4ac4ea78", + "worker_process_ms": 1201.965200016275 + }, + "ooxml": { + "backend": "ooxml", + "import_ms": 229.1276000905782, + "first_extract_ms": 8.505499921739101, + "cold_in_process_ms": 238.88249997980893, + "repeated_extract_ms": [ + 7.120700087398291, + 6.683700019493699, + 5.587999941781163, + 5.924399942159653, + 5.377799971029162 + ], + "median_extract_ms": 5.924399942159653, + "peak_rss_bytes": 41512960, + "workbook_opens": 0, + "archive_opens": 1, + "loaded": [], + "fallback_reason": null, + "output_sha256": "1630d9cc31264fe0ab7291794b97bc47b7dc9c545f1f2b69dfa8f5eb4ac4ea78", + "worker_process_ms": 513.9168000314385 + } + } + ] +} diff --git a/benchmark/issue150-results.md b/benchmark/issue150-results.md new file mode 100644 index 00000000..409014ca --- /dev/null +++ b/benchmark/issue150-results.md @@ -0,0 +1,48 @@ +# Issue #150 light backend comparison + +Windows / Python 3.11.13, five repeated extractions in separate fresh processes +for each backend. The baseline is the preserved `_run_openpyxl_pipeline`, and +the candidate is the default public light pipeline. Both use the same installed +dependencies and input bytes. Default include flags are used. The full emitted +workbook model SHA-256 matches for every input. + +| Fixture | Repeated extraction median, openpyxl / OOXML (ms) | Cold in-process, openpyxl / OOXML (ms) | Peak working set, openpyxl / OOXML (MiB) | +| --- | --- | --- | --- | +| small | 10.8 / 5.0 | 983.4 / 428.1 | 64.4 / 39.6 | +| large (60,000 cells) | 1017.5 / 607.5 | 1626.2 / 1060.1 | 109.3 / 71.8 | +| many-sheet (24 sheets) | 107.7 / 74.1 | 839.9 / 333.2 | 74.5 / 42.7 | +| sparse | 51.3 / 16.9 | 823.0 / 254.0 | 75.3 / 39.7 | +| style-heavy | 136.0 / 110.5 | 902.1 / 460.5 | 73.6 / 46.3 | +| table-heavy (20 tables) | 86.8 / 56.3 | 823.3 / 313.1 | 75.3 / 44.0 | +| tracked shape/connector diagram | 16.2 / 5.9 | 842.9 / 238.9 | 64.6 / 39.6 | + +All candidate runs opened one ZIP and zero openpyxl workbooks, compared with +two ZIPs and one openpyxl workbook on the compatibility baseline. Candidate +runs imported none of openpyxl, pandas, SciPy or xlwings and used no fallback. +The default direct table path uses Python clustering even when SciPy is installed; +the compatibility baseline retains its default automatic SciPy selection. + +Cold in-process time includes pipeline import, setup and first extraction; it +excludes OS process creation and serialization. `worker_process_ms` in the raw +data includes repeats and instrumentation and is not a cold-start measurement. +Peak memory is the process lifetime working-set high-water mark after repeated +extraction, captured before resource-count instrumentation. These local +sequential measurements establish the acceptance target for these fixtures, +not a universal speed or memory guarantee. Unsupported workbook fallback may +be slower because it runs both readers. + +Raw measurements, hashes and environment metadata: +[issue150-2026-10-03-windows.json](baselines/issue150-2026-10-03-windows.json). + +## Reproduce + +Use a new fixture directory; generation refuses to overwrite existing fixtures. + +```powershell +rtk uv run python -m benchmark.performance --generate-fixtures tasks/issue150-fixtures +rtk uv run python -m benchmark.issue150_light --input tasks/issue150-fixtures/small.xlsx tasks/issue150-fixtures/large.xlsx tasks/issue150-fixtures/many-sheet.xlsx tasks/issue150-fixtures/sparse.xlsx tasks/issue150-fixtures/style-heavy.xlsx tasks/issue150-fixtures/table-heavy.xlsx sample/flowchart/sample-shape-connector.xlsx --repeats 5 --output benchmark/baselines/issue150-2026-10-03-windows.json +``` + +The implementation caches date/duration classification per style and reuses +parsed coordinates. Fully qualified child tags avoid repeated XPath processing +without changing the safe XML parser or cached-value semantics. diff --git a/benchmark/issue150_light.py b/benchmark/issue150_light.py new file mode 100644 index 00000000..6eaa0d4f --- /dev/null +++ b/benchmark/issue150_light.py @@ -0,0 +1,190 @@ +"""Compare direct OOXML light extraction with its preserved openpyxl pipeline.""" + +from __future__ import annotations + +import argparse +from datetime import UTC, datetime +import hashlib +from functools import wraps +from importlib.metadata import PackageNotFoundError, version +import json +from pathlib import Path +import platform +import statistics +import subprocess +import sys +from time import perf_counter +from typing import Any + + +def worker(path: Path, backend: str, repeats: int) -> dict[str, Any]: + started = perf_counter() + from exstruct.core import pipeline, workbook + from exstruct.core.ooxml_session import OoxmlExtractionSession + + import_ms = (perf_counter() - started) * 1000 + from benchmark.performance import peak_rss_bytes + + inputs = pipeline.resolve_extraction_inputs( + path, + mode="light", + include_cell_links=None, + include_print_areas=None, + include_auto_page_breaks=False, + include_colors_map=None, + include_default_background=False, + ignore_colors=None, + include_formulas_map=None, + include_merged_cells=None, + include_merged_values_in_rows=True, + ) + runner = ( + pipeline.run_extraction_pipeline + if backend == "ooxml" + else pipeline._run_openpyxl_pipeline + ) + start = perf_counter() + result = runner(inputs) + first_ms = (perf_counter() - start) * 1000 + cold_ms = (perf_counter() - started) * 1000 + samples = [] + for _ in range(repeats): + start = perf_counter() + runner(inputs) + samples.append((perf_counter() - start) * 1000) + peak = peak_rss_bytes() + loaded = [ + name + for name in ("openpyxl", "xlwings", "scipy", "pandas") + if name in sys.modules + ] + encoded = json.dumps( + result.workbook.model_dump(mode="json"), ensure_ascii=True, sort_keys=True + ) + loads = 0 + archives = 0 + original_load = workbook.load_workbook + original_archive = OoxmlExtractionSession.archive + from zipfile import ZipFile + + original_zip_init = ZipFile.__init__ + + @wraps(original_load) + def load(*args: Any, **kwargs: Any) -> Any: + nonlocal loads + loads += 1 + return original_load(*args, **kwargs) + + def zip_init(self: ZipFile, *args: Any, **kwargs: Any) -> None: + nonlocal archives + archives += 1 + original_zip_init(self, *args, **kwargs) + + # Count all ZIP constructors, including the previous standalone rich parser. + workbook.load_workbook = load + ZipFile.__init__ = zip_init + try: + profiled = runner(inputs) + finally: + workbook.load_workbook = original_load + ZipFile.__init__ = original_zip_init + assert profiled.workbook == result.workbook + assert OoxmlExtractionSession.archive is original_archive + return { + "backend": backend, + "import_ms": import_ms, + "first_extract_ms": first_ms, + "cold_in_process_ms": cold_ms, + "repeated_extract_ms": samples, + "median_extract_ms": statistics.median(samples), + "peak_rss_bytes": peak, + "workbook_opens": loads, + "archive_opens": archives, + "loaded": loaded, + "fallback_reason": result.state.fallback_reason, + "output_sha256": hashlib.sha256(encoded.encode()).hexdigest(), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, nargs="+", required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--worker", choices=["ooxml", "openpyxl"]) + args = parser.parse_args() + if args.repeats < 1: + parser.error("--repeats must be positive") + if args.worker: + print(json.dumps(worker(args.input[0], args.worker, args.repeats))) + return + results = [] + for path in args.input: + measurements = {} + # Separate processes ensure compatibility imports cannot contaminate OOXML. + for backend in ("openpyxl", "ooxml"): + started = perf_counter() + completed = subprocess.run( + [ + sys.executable, + "-m", + "benchmark.issue150_light", + "--worker", + backend, + "--input", + str(path), + "--repeats", + str(args.repeats), + ], + capture_output=True, + text=True, + encoding="utf-8", + check=True, + timeout=300, + ) + measurement = json.loads(completed.stdout) + measurement["worker_process_ms"] = (perf_counter() - started) * 1000 + measurements[backend] = measurement + if ( + measurements["openpyxl"]["output_sha256"] + != measurements["ooxml"]["output_sha256"] + ): + raise RuntimeError(f"Output mismatch: {path}") + results.append( + { + "input": str(path), + "input_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + **measurements, + } + ) + from benchmark.performance import source_metadata + + dependencies = {} + for name in ("exstruct", "openpyxl", "numpy", "scipy", "defusedxml", "pydantic"): + try: + dependencies[name] = version(name) + except PackageNotFoundError: + dependencies[name] = None + output = ( + json.dumps( + { + "recorded_at": datetime.now(UTC).isoformat(), + "python": sys.version, + "platform": platform.platform(), + "source": source_metadata(10), + "dependencies": dependencies, + "results": results, + }, + indent=2, + ) + + "\n" + ) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(output, encoding="utf-8") + else: + print(output, end="") + + +if __name__ == "__main__": + main() diff --git a/benchmark/performance.py b/benchmark/performance.py index 2ebfd01d..1a045f5e 100644 --- a/benchmark/performance.py +++ b/benchmark/performance.py @@ -135,7 +135,9 @@ def profile_extract() -> WorkbookData: else serialized ) with ExitStack() as stack: - recorder.install(stack) + recorder.install( + stack, direct_ooxml=mode == "light" and not profile_all_features + ) started = perf_counter() profiled_workbook = profile_extract() profile_ms = (perf_counter() - started) * 1000 diff --git a/benchmark/performance_profile.py b/benchmark/performance_profile.py index d1778dc8..9a5bba7e 100644 --- a/benchmark/performance_profile.py +++ b/benchmark/performance_profile.py @@ -47,13 +47,12 @@ def wrapped(*args: object, **kwargs: object) -> object: return wrapped - def install(self, stack: ExitStack) -> None: + def install(self, stack: ExitStack, *, direct_ooxml: bool = False) -> None: """Patch live call sites after lazy extraction imports have completed.""" - import openpyxl - from exstruct.core import integrate, pipeline from exstruct.core.backends.ooxml_backend import OoxmlRichBackend from exstruct.core.backends.openpyxl_backend import OpenpyxlBackend + from exstruct.core.ooxml_session import OoxmlExtractionSession methods = { "extract_cells": "cell_extraction", @@ -63,12 +62,15 @@ def install(self, stack: ExitStack) -> None: "extract_merged_cells": "merged_cell_extraction", "detect_tables": "table_detection", } + owner = OoxmlExtractionSession if direct_ooxml else OpenpyxlBackend for method, stage in methods.items(): + if direct_ooxml and method == "extract_colors_map": + continue stack.enter_context( patch.object( - OpenpyxlBackend, + owner, method, - self.timed(stage, getattr(OpenpyxlBackend, method)), + self.timed(stage, getattr(owner, method)), ) ) functions = { @@ -95,15 +97,18 @@ def install(self, stack: ExitStack) -> None: ) ) - # Match function identity to include imported aliases and pandas' reader. - original_load = openpyxl.load_workbook - wrapped_load = self.timed("workbook_parsing", original_load) - for module in list(sys.modules.values()): - if module is None: - continue - for name, value in list(vars(module).items()): - if value is original_load: - stack.enter_context(patch.object(module, name, wrapped_load)) + if not direct_ooxml: + import openpyxl + + # Match function identity to include imported compatibility aliases. + original_load = openpyxl.load_workbook + wrapped_load = self.timed("workbook_parsing", original_load) + for module in list(sys.modules.values()): + if module is None: + continue + for name, value in list(vars(module).items()): + if value is original_load: + stack.enter_context(patch.object(module, name, wrapped_load)) original_zip_init = zipfile.ZipFile.__init__ diff --git a/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md b/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md index 1ba90e27..b3a63f41 100644 --- a/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md +++ b/dev-docs/adr/ADR-0011-extraction-scoped-workbook-reuse-and-direct-cell-reading.md @@ -13,20 +13,24 @@ production cell reader adds a substantial dependency without providing a public abstraction. Independent path-based helpers also parse the same OOXML workbook repeatedly, including from nested table detection helpers. -ADR-0010 already defines light as a pure-Python rich OOXML baseline. This -decision improves its implementation while retaining that mode boundary, -serialization, coordinate conventions and fallback behavior. It does not -introduce a new public extraction backend or change COM-first/pre-COM ordering. +ADR-0010 defines light as a pure-Python rich OOXML baseline. This decision +records direct openpyxl reading and workbook ownership for paths that select +the compatibility backend. Issue #150 selects direct OOXML for normal +`.xlsx` / `.xlsm` light extraction; ADR-0013 records that light-only refinement. +The output model and coordinate conventions remain unchanged. ## 決定 -- Read `.xlsx` and `.xlsm` cells directly through openpyxl worksheet iteration; - build `CellRow` without a pandas DataFrame. Preserve the previous reader's - observable normalization, empty-cell filtering and cached-formula behavior. +- On the openpyxl compatibility path, read `.xlsx` and `.xlsm` cells directly + through worksheet iteration; build `CellRow` without a pandas DataFrame. + Normal `.xlsx` / `.xlsm` light extraction uses the direct OOXML path in + ADR-0013. Preserve the previous reader's observable normalization, + empty-cell filtering and cached-formula behavior on the openpyxl path. - Retain `.xls` cell reading through a lazily imported direct xlrd reader. This preserves the public input format without retaining pandas or requiring Excel COM for cell reading. It does not add OOXML artifacts to BIFF workbooks. -- Scope openpyxl workbook ownership to one extraction invocation. Compatible +- Scope openpyxl workbook ownership to one extraction invocation whenever the + openpyxl path is selected. Compatible stages reuse a regular `data_only=True` workbook. Open a separate `data_only=False` workbook only when openpyxl formula extraction is requested. - Close owned workbooks when the extraction finishes or raises, including @@ -44,7 +48,9 @@ introduce a new public extraction backend or change COM-first/pre-COM ordering. ## 影響 - Direct cell construction removes production DataFrame allocation and the - pandas runtime requirement; pandas may remain in development/benchmark tools. + pandas runtime requirement. The normal light OOXML path also avoids importing + openpyxl; openpyxl remains installed for compatibility fallback and paths + that select it. Pandas may remain in development/benchmark tools. - Per-invocation ownership reduces parsing cost without stale data across calls or cross-extraction resource sharing. Formula extraction still needs a second workbook because its load semantics differ from cached cell values. @@ -74,9 +80,11 @@ introduce a new public extraction backend or change COM-first/pre-COM ordering. `src/exstruct/core/pipeline.py`, `src/exstruct/core/openpyxl_session.py`, `src/exstruct/core/cell_types.py`, and `src/exstruct/core/backends/`. - Related specs: `dev-docs/specs/excel-extraction.md`, + `dev-docs/specs/ooxml-core-extraction.md`, `dev-docs/specs/extraction-performance.md`, and `docs/api.md`. - Related decisions: ADR-0002 fallback policy, ADR-0003 serialization policy, - and ADR-0010 light-mode responsibility boundary. + ADR-0010 light-mode responsibility boundary, and ADR-0013 light OOXML backend + selection. ## Supersedes diff --git a/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md b/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md index a0b83cf7..53c0bbf7 100644 --- a/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md +++ b/dev-docs/adr/ADR-0012-optional-scipy-border-clustering.md @@ -25,6 +25,11 @@ existing Python fallback on 2026-10-03. `auto` attempts SciPy, `python` uses the existing BFS, and the legacy `numpy` name attempts SciPy-backed labeling. Import or accelerated execution failure still falls back to Python. Do not add a new alias or change defaults. +- The normal direct OOXML `light` table-heuristic path is a narrow exception: + it passes `cluster_backend="python"` per call so that path does not import + SciPy. This does not read or mutate `EXSTRUCT_BORDER_CLUSTER_BACKEND`. + Other modes and the openpyxl compatibility path retain the environment-based + selection above. - Keep sparse-set BFS as an internal evaluation candidate. Do not promote it to production selection based on these results. @@ -53,11 +58,14 @@ existing Python fallback on 2026-10-03. - Code: `src/exstruct/core/cells.py`, `pyproject.toml`, `uv.lock`, `benchmark/issue148_clustering.py`, and `benchmark/issue148_scipy_free.py`. - Related specs: `docs/api.md`, `dev-docs/specs/excel-extraction.md`, + `dev-docs/specs/ooxml-core-extraction.md`, `dev-docs/specs/extraction-performance.md`, and `benchmark/issue148-results.md`. - Measurements: `benchmark/baselines/issue148-2026-10-03-windows.json` records raw timings, input hashes, output hashes, dependency versions and source state. - Related decisions: ADR-0010 retains the light OOXML baseline; ADR-0011 treats - direct cell reading and extraction-scoped resources separately. + direct cell reading and extraction-scoped resources separately; ADR-0013 + selects the default direct backend for light OOXML extraction and records its + per-call Python clustering exception. ## Supersedes diff --git a/dev-docs/adr/ADR-0013-default-ooxml-backend-for-light-extraction.md b/dev-docs/adr/ADR-0013-default-ooxml-backend-for-light-extraction.md new file mode 100644 index 00000000..a8903919 --- /dev/null +++ b/dev-docs/adr/ADR-0013-default-ooxml-backend-for-light-extraction.md @@ -0,0 +1,100 @@ +# ADR-0013: Default OOXML Backend for Light Extraction + +## 状態 + +`proposed` + +## 背景 + +Issue #149 added a reusable OOXML session for workbook core data and rich +drawings. Issue #150 selects that implementation for the normal `.xlsx` / +`.xlsm` `light` path. This changes backend priority and failure handling, so it +requires a policy decision beyond the capability work recorded in ADR-0011. + +ADR-0011 records direct openpyxl cell reading and extraction-scoped workbook +ownership. Its workbook-reuse and compatibility guarantees remain useful, but +the normal `light` OOXML path needs to avoid importing openpyxl, pandas, SciPy, +and xlwings while preserving the existing output model and legacy overrides. +Unsupported OOXML constructs also need a defined recovery rule: partial direct +results cannot safely be combined with results from a second reader. + +## 決定 + +- For `.xlsx` and `.xlsm` in `light` mode, select the direct OOXML pipeline by + default. One `OoxmlExtractionSession` owns one ZIP archive shared by core + workbook extraction and rich shapes/charts through final model construction. +- The normal supported path must not import openpyxl, pandas, SciPy, or xlwings. + Keep scalar normalization and table-candidate detection on the direct path + independent of those optional/runtime readers. The direct light table + heuristic explicitly passes `cluster_backend="python"` per call; it does not + change `EXSTRUCT_BORDER_CLUSTER_BACKEND`. Other modes and compatibility + extraction retain ADR-0012's configured backend selection. +- If any uncaught failure or unsupported construct occurs in any direct OOXML + stage, discard all partial OOXML results, close the session in `finally`, and + restart the complete extraction through `_run_openpyxl_pipeline`. +- Select that complete compatibility pipeline when `colors_map` is explicitly + requested or a legacy pipeline/helper/workbook override is active. Known + unsupported options and overrides should be detected before opening the ZIP + where possible. +- Log the compatibility restart as a warning with + `FallbackReason.OOXML_COMPATIBILITY` (`ooxml_compatibility`) and carry the + reason in the extraction state when compatibility completes normally. If the + compatibility pipeline takes its own fallback, preserve that later reason + in state and retain the initial OOXML warning in the log. +- Preserve existing per-sheet best-effort drawing behavior: a drawing failure + isolated to one sheet may omit that sheet's affected rich artifacts while + healthy sheets and core extraction continue. +- Preserve the current output semantics and models. Other extraction modes + and `.xls` selection/behavior remain unchanged. +- ADR-0011 continues to govern openpyxl resource ownership and direct reading + for the compatibility path and other applicable extraction paths. This ADR + refines only normal `.xlsx` / `.xlsm` `light` backend selection, so it does + not supersede ADR-0011 as a whole. + +## 影響 + +- Normal supported `light` extraction avoids loading four heavyweight reader + or acceleration modules and reuses one archive for core and rich extraction. +- Openpyxl remains necessary for compatibility fallback and existing paths. + Unsupported workbooks can pay the cost of a complete second extraction after + the direct attempt, although partial results are never mixed. +- Direct OOXML readers and table heuristics must preserve the established + normalization, table-candidate, rich-artifact, and serialized-output + contracts. Parity regressions and an explicit import-boundary check are + required. +- The recorded Issue #150 Windows run produced matching serialized-output + hashes for all seven inputs. The direct pipeline had lower cold in-process + time, warm median and peak working set on each input; the report and raw + baseline contain the measurements and their limits. These are local + observations, not universal performance guarantees. + +## 根拠 + +- Tests: `tests/core/test_light_ooxml_pipeline.py` covers previous-pipeline + parity, tracked sample parity, one-ZIP ownership and closure, whole-result + restart after a stage failure, colors opt-in compatibility, and unchanged + selection for other modes. `tests/core/test_backend_import_isolation.py` + covers the normal-path import boundary. +- Code: `src/exstruct/core/pipeline.py`, + `src/exstruct/core/ooxml_session.py`, + `src/exstruct/core/ooxml_scalars.py`, + `src/exstruct/core/backends/ooxml_backend.py`, and + `src/exstruct/errors.py`. +- Related specs: `dev-docs/specs/excel-extraction.md`, + `dev-docs/specs/ooxml-core-extraction.md`, + `dev-docs/specs/extraction-performance.md`, `docs/api.md`, `docs/cli.md`, + `README.md`, and `README.ja.md`. +- Benchmark: `benchmark/issue150_light.py`; final results are summarized in + `benchmark/issue150-results.md` and recorded at + `benchmark/baselines/issue150-2026-10-03-windows.json`. +- Related decisions: ADR-0010 defines the light rich-output boundary; + ADR-0011 defines extraction-scoped openpyxl ownership and compatibility; + ADR-0012 defines the optional SciPy selection and the direct-light exception. + +## Supersedes + +- None + +## Superseded by + +- None diff --git a/dev-docs/adr/README.md b/dev-docs/adr/README.md index 8f8f8666..11d91c74 100644 --- a/dev-docs/adr/README.md +++ b/dev-docs/adr/README.md @@ -47,3 +47,4 @@ ADRs record what was decided, under which constraints, and which trade-offs were | `ADR-0010` | Light Mode as the Pure-Python Rich OOXML Baseline | `accepted` | `extraction` | | `ADR-0011` | Extraction-Scoped Workbook Reuse and Direct Cell Reading | `proposed` | `extraction` | | `ADR-0012` | Optional SciPy Border Clustering | `proposed` | `performance` | +| `ADR-0013` | Default OOXML Backend for Light Extraction | `proposed` | `backend` | diff --git a/dev-docs/adr/decision-map.md b/dev-docs/adr/decision-map.md index b85cb0ca..81b1a82e 100644 --- a/dev-docs/adr/decision-map.md +++ b/dev-docs/adr/decision-map.md @@ -10,6 +10,7 @@ This document is a human-readable map for navigating ADRs by domain. - `ADR-0010` Light Mode as the Pure-Python Rich OOXML Baseline (`accepted`) - `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) - `ADR-0012` Optional SciPy Border Clustering (`proposed`) +- `ADR-0013` Default OOXML Backend for Light Extraction (`proposed`) ## mode @@ -23,10 +24,12 @@ This document is a human-readable map for navigating ADRs by domain. - `ADR-0010` Light Mode as the Pure-Python Rich OOXML Baseline (`accepted`) - `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) - `ADR-0012` Optional SciPy Border Clustering (`proposed`) +- `ADR-0013` Default OOXML Backend for Light Extraction (`proposed`) ## fallback - `ADR-0002` Rich Backend Fallback Policy (`accepted`) +- `ADR-0013` Default OOXML Backend for Light Extraction (`proposed`) ## schema @@ -45,11 +48,13 @@ This document is a human-readable map for navigating ADRs by domain. - `ADR-0010` Light Mode as the Pure-Python Rich OOXML Baseline (`accepted`) - `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) - `ADR-0012` Optional SciPy Border Clustering (`proposed`) +- `ADR-0013` Default OOXML Backend for Light Extraction (`proposed`) ## performance - `ADR-0011` Extraction-Scoped Workbook Reuse and Direct Cell Reading (`proposed`) - `ADR-0012` Optional SciPy Border Clustering (`proposed`) +- `ADR-0013` Default OOXML Backend for Light Extraction (`proposed`) ## mcp diff --git a/dev-docs/adr/index.yaml b/dev-docs/adr/index.yaml index 6917efcc..10affed4 100644 --- a/dev-docs/adr/index.yaml +++ b/dev-docs/adr/index.yaml @@ -170,6 +170,7 @@ adrs: superseded_by: [] related_specs: - dev-docs/specs/excel-extraction.md + - dev-docs/specs/ooxml-core-extraction.md - dev-docs/specs/extraction-performance.md - docs/api.md - id: ADR-0012 @@ -187,5 +188,27 @@ adrs: related_specs: - docs/api.md - dev-docs/specs/excel-extraction.md + - dev-docs/specs/ooxml-core-extraction.md - dev-docs/specs/extraction-performance.md - benchmark/issue148-results.md + - id: ADR-0013 + title: Default OOXML Backend for Light Extraction + status: proposed + path: dev-docs/adr/ADR-0013-default-ooxml-backend-for-light-extraction.md + primary_domain: backend + domains: + - backend + - extraction + - fallback + - compatibility + - performance + supersedes: [] + superseded_by: [] + related_specs: + - dev-docs/specs/excel-extraction.md + - dev-docs/specs/ooxml-core-extraction.md + - dev-docs/specs/extraction-performance.md + - docs/api.md + - docs/cli.md + - README.md + - README.ja.md diff --git a/dev-docs/adr/reviews/ADR-0013-2026-10-03.md b/dev-docs/adr/reviews/ADR-0013-2026-10-03.md new file mode 100644 index 00000000..13a226ff --- /dev/null +++ b/dev-docs/adr/reviews/ADR-0013-2026-10-03.md @@ -0,0 +1,77 @@ +# ADR-0013 review and validation + +Issue: [#150](https://github.com/harumiWeb/exstruct/issues/150) + +## Structural review + +`adr-linter`: no findings. ADR-0013 has a valid `proposed` status, required +sections, concrete tests/code/spec evidence, balanced consequences and explicit +supersede fields. The later clarification about compatibility's own fallback +reason preserves those requirements and matches the implementation. + +`adr-indexer`: README, index.yaml and decision-map.md were synchronized. The +13 ADR IDs, paths, status/domain entries and supersede relationships are +consistent. ADR-0013 remains proposed. + +## Design review + +`adr-reviewer` verdict: `ready`. Findings: none. Open questions: none. + +Scope: ADR-0013 and the amended ADR-0011/0012; public API/CLI docs and English/ +Japanese README; `excel-extraction.md`, `ooxml-core-extraction.md`, +`extraction-performance.md`; `pipeline.py`, `ooxml_session.py`, `cells.py`, +and the light/scalar regression tests; Issue #150 and its benchmark report. + +`adr-reconciler`: no findings; recommended action `no-action`. The issue's +requested backend priority, whole-result compatibility restart, preserved +per-sheet drawing behavior and direct-light Python clustering exception agree +across ADR, specs, source and tests. The reviewer read the evidence and did not +independently rerun the full suite or performance measurements. + +Residual risks: fallback workbooks can require two reader attempts; supported +rich artifacts remain best-effort. Windows benchmark observations do not imply +universal latency or memory guarantees. `ready` is a review verdict, not merge +approval or an accepted ADR status. + +## Implementation review follow-up + +The separate code reviewer confirmed two P2 parity defects in the initial +implementation: unquoted coordinate-like sheet names were shifted as cell +references in shared formulas, and uppercase elapsed-time formats were treated +as dates. Both were corrected before completion: + +- Unquoted sheet qualifiers, non-ASCII shared names and references crossing + worksheet bounds request whole compatibility extraction. Ordinary A1, + quoted sheet names and string literals retain direct translation support. +- Elapsed-time format matching is case-insensitive; the numeric cached value + and merged anchor retain the established duration representation. + +Evidence: `tests/core/test_light_ooxml_pipeline.py` compares actual shared +formula followers and uppercase duration output with the compatibility +pipeline; `tests/core/test_ooxml_scalars.py` covers capability signaling. + +## Local validation + +- Full suite: 1,077 passed, 12 COM/render/LibreOffice tests deselected; coverage + 83.97% passed the 80% gate. +- Ruff, strict mypy (89 source files) and `git diff --check` passed. +- Real subprocess tests deny imports of openpyxl, pandas, SciPy and xlwings + while extracting cells, border-based tables, charts and a tracked diagram. +- Complete-model parity covers the tracked XLSX samples, sparse/multiple sheets, + formulas, links, merges, print areas, tables and rich artifacts. XLSM fixtures + preserve the entire input checksum, including an opaque VBA member. +- Seven benchmark inputs matched full-model JSON hashes; cold in-process time, + repeated extraction median and peak working set improved on each. Direct + runs opened one ZIP and zero openpyxl workbooks without fallback. + +Commands: + +```powershell +rtk uv run --extra all pytest -m "not com and not render and not libreoffice" -q --cov=exstruct --cov-report=term --cov-fail-under=80 +rtk uv run ruff check . --no-fix +rtk uv run --extra all mypy src/exstruct --strict +rtk git diff --check +``` + +Performance evidence: [Issue #150 results](../../../benchmark/issue150-results.md) +and [raw baseline](../../../benchmark/baselines/issue150-2026-10-03-windows.json). diff --git a/dev-docs/architecture/pipeline.md b/dev-docs/architecture/pipeline.md index 2703e682..fef6c8b6 100644 --- a/dev-docs/architecture/pipeline.md +++ b/dev-docs/architecture/pipeline.md @@ -6,7 +6,7 @@ to convert Excel workbooks into **semantically structured JSON**. This design achieves the following. - Separation of Excel COM-dependent logic from non-dependent logic -- Future extensibility to direct OpenXML/XML parsing +- Direct OOXML reading for the normal light path on .xlsx / .xlsm - Stable output for RAG/LLM use cases --- @@ -15,23 +15,27 @@ This design achieves the following. ```mermaid sequenceDiagram - participant Client - participant Pipeline - participant OpenpyxlBackend + participant Client + participant Pipeline + participant OoxmlSession + participant OpenpyxlBackend participant RichBackend participant Modeling Client->>Pipeline: extract() - Pipeline->>OpenpyxlBackend: pre_extract() - OpenpyxlBackend-->>Pipeline: cells / tables / print_areas - - alt Rich backend available - Pipeline->>RichBackend: extract_shapes(mode=...) - RichBackend-->>Pipeline: shapes - Pipeline->>RichBackend: extract_charts(mode=...) - RichBackend-->>Pipeline: charts - else runtime unavailable - Pipeline->>Pipeline: log_fallback() + alt supported light .xlsx/.xlsm + Pipeline->>OoxmlSession: core + rich extraction (one ZIP) + OoxmlSession-->>Pipeline: cells / tables / print_areas / shapes / charts + opt unsupported construct or direct-stage failure + Pipeline->>Pipeline: discard partial result; close session + Pipeline->>OpenpyxlBackend: restart complete compatibility pipeline + OpenpyxlBackend-->>Pipeline: complete compatibility result + end + else other modes, .xls, or compatibility selection + Pipeline->>OpenpyxlBackend: selected extraction path + OpenpyxlBackend-->>Pipeline: core extraction + Pipeline->>RichBackend: selected rich extraction + RichBackend-->>Pipeline: rich artifacts end Pipeline->>Modeling: integrate() @@ -43,11 +47,18 @@ The processing order is as follows. `RichBackend` in this diagram refers to the conceptual rich-extraction layer; the concrete implementations are `OoxmlRichBackend`, `ComRichBackend`, and `LibreOfficeRichBackend`. -1. **Pipeline** assembles the execution plan -2. **Openpyxl Backend** performs pre-analysis (cells, tables, print areas) -3. **Rich Backend** extracts shapes/charts if available. Here, `RichBackend` is the conceptual layer and `light` uses `OoxmlRichBackend`, while COM-backed modes use `ComRichBackend` and optional LibreOffice enrichment uses `LibreOfficeRichBackend`. -4. **Modeling** integrates the results into WorkbookData / SheetData -5. Output in the requested format (JSON / YAML / TOON) +1. **Pipeline** selects the mode and backend. +2. Supported .xlsx / .xlsm light extraction reads core and rich OOXML data + through one OoxmlExtractionSession ZIP. +3. An unsupported construct or uncaught failure at any direct stage discards + the partial result, closes the session, and restarts the full openpyxl + compatibility pipeline. colors_map opt-in and active legacy overrides + select compatibility directly. +4. Other modes and .xls retain their existing backend selection. The + conceptual rich layer includes OoxmlRichBackend, ComRichBackend, and + LibreOfficeRichBackend. +5. **Modeling** integrates the result into WorkbookData / SheetData; output is + serialized in the requested format (JSON / YAML / TOON). --- @@ -59,18 +70,17 @@ Pipeline is the **orchestrator**. - Selects backends - Controls fallback paths - Manages intermediate artifacts -- Owns one extraction-scoped openpyxl session across pre-analysis, COM/fallback - processing and final model construction. Backends consume that session rather - than opening the same workbook for each feature or worksheet. +- Owns one extraction-scoped OOXML session for normal light .xlsx / .xlsm + processing, or one openpyxl session when the compatibility/other path is + selected. Resources remain scoped through final model construction. Pipeline is designed to **never read Excel content directly**. -`OpenpyxlExtractionSession` lazily owns regular workbook variants through an -`ExitStack`: cached values (`data_only=True`) are shared by cells, hyperlinks, -print areas, colors, merged cells and table detection; formula text -(`data_only=False`) is opened only when requested. Path-based standalone helpers -remain wrappers with their own bounded lifetime. The session is not a global -cache and must not outlive one extraction. +OoxmlExtractionSession shares one ZIP between core and rich light extraction. +When openpyxl compatibility is selected, OpenpyxlExtractionSession owns the +regular workbook variants: cached values (data_only=True) are shared by +compatible stages, and formula text (data_only=False) is opened only when +requested. Neither session is a global cache or outlives one extraction. --- @@ -81,6 +91,7 @@ Backend defines **how Excel is read**. | Backend | Responsibilities | | ---------------------- | ------------------------------------------------- | | OpenpyxlBackend | Cells / tables / print areas / colors map | +| OoxmlExtractionSession | Light-mode OOXML core data and shared ZIP lifetime | | ComBackend | COM-only print areas / auto page breaks / maps | | OoxmlRichBackend | Pure-Python OOXML shapes / connectors / charts | | ComRichBackend | Shapes / arrows / charts / SmartArt via Excel COM | @@ -100,11 +111,17 @@ All of these can be added **without major changes to the Pipeline**. ## Fallback Design -When COM or LibreOffice runtime is unavailable, the following must be respected. - -- Do not take down the entire process with an exception -- Reuse openpyxl results as much as possible -- Record the fallback reason explicitly +Fallback behavior depends on the selected path. + +- A direct light OOXML stage failure or unsupported construct discards every + partial result, closes the session in finally, and restarts the complete + openpyxl compatibility pipeline with the ooxml_compatibility warning. +- A drawing failure isolated to one sheet retains the existing best-effort + behavior and does not discard healthy sheet output. +- colors_map opt-in and active legacy overrides select compatibility; other + modes and .xls retain their existing routing. +- COM/LibreOffice fallback behavior remains as defined by the mode contract. +- Record fallback reasons explicitly through FallbackReason. This is an intentional design that assumes **batch processing, CI, and automation**. diff --git a/dev-docs/roadmap.md b/dev-docs/roadmap.md index aaf180f5..8de71401 100644 --- a/dev-docs/roadmap.md +++ b/dev-docs/roadmap.md @@ -1,13 +1,14 @@ # Extraction roadmap -This records the scope already established by the extraction issues. It does -not introduce a new public mode or approve a default-backend migration. +This records the extraction work established by the issues. Current contracts +are defined by the specifications and ADRs linked below. | Milestone | Status | Contract and evidence | | --- | --- | --- | -| Core OOXML session foundation (Issue #149, PR #154) | Implemented; pending merge | Direct cells, strings, formulas, links, merges, names, print areas and explicit tables; shared ZIP/relationships for optional drawings. See [core OOXML specification](specs/ooxml-core-extraction.md) and `tests/core/test_ooxml_extraction_session.py`. | -| Default-path OOXML migration | Deferred to a separate issue | Current openpyxl pipeline remains selected. Evaluate parity and capability gaps, then assess the migration's ADR and public-contract impact before changing selection. | -| OOXML table heuristics | Outside Issue #149 | Explicit tables are available now; border/style-based heuristics remain on the existing backend. Any extension needs its own scope and parity evidence. | +| Core OOXML session foundation (Issue #149, PR #154) | Merged | Direct cells, strings, formulas, links, merges, names, print areas and explicit tables; shared ZIP/relationships for optional drawings. See [core OOXML specification](specs/ooxml-core-extraction.md) and tests/core/test_ooxml_extraction_session.py. | +| Default light OOXML pipeline (Issue #150) | Implemented and locally verified; pending merge | .xlsx / .xlsm light defaults to one shared OOXML session for core and rich output. Unsupported input, colors_map, or legacy overrides restart the complete openpyxl compatibility pipeline. See [ADR-0013](adr/ADR-0013-default-ooxml-backend-for-light-extraction.md), [extraction spec](specs/excel-extraction.md), tests/core/test_light_ooxml_pipeline.py, and [performance results](specs/extraction-performance.md). | +| Light OOXML table heuristics (Issue #150) | Implemented and locally verified; pending merge | Preserve table-candidate behavior with direct OOXML heuristics. The direct light path selects Python BFS per call, without changing the clustering environment variable; see [ADR-0012](adr/ADR-0012-optional-scipy-border-clustering.md). | +| Issue #150 performance comparison | Recorded | benchmark/issue150_light.py compared seven inputs in fresh openpyxl and OOXML workers; all serialized-output hashes matched. See [measured results](specs/extraction-performance.md) and benchmark/baselines/issue150-2026-10-03-windows.json. | -Current behavior is defined by [Excel extraction](specs/excel-extraction.md), -not by the roadmap status labels. +Current behavior is defined by [Excel extraction](specs/excel-extraction.md) +and [core OOXML extraction](specs/ooxml-core-extraction.md). diff --git a/dev-docs/specs/excel-extraction.md b/dev-docs/specs/excel-extraction.md index 4fde3657..70a6e52e 100644 --- a/dev-docs/specs/excel-extraction.md +++ b/dev-docs/specs/excel-extraction.md @@ -4,37 +4,54 @@ This document summarizes the current specification for Excel extraction processi ## Overall Flow -1. `resolve_extraction_inputs` normalizes include_* and mode -2. Pre-com (openpyxl) retrieves cells/print_areas/formulas_map/colors_map/merged_cells -3. `light` uses the OOXML rich backend for best-effort shapes/connectors/charts on `.xlsx/.xlsm` -4. `libreoffice` seeds the same OOXML rich baseline and then uses the LibreOffice backend for optional best-effort enrichment on `.xlsx/.xlsm` -5. `standard` / `verbose` use COM (xlwings) to retrieve shapes/charts/auto_page_breaks -6. When COM succeeds, colors_map is overwritten with COM results -7. When the rich backend fails, cells+table_candidates are preserved, and pre-com artifacts (print_areas / formulas_map / colors_map / merged_cells) are also retained according to their flags +1. `resolve_extraction_inputs` normalizes include_* and mode. +2. For `light` on `.xlsx` / `.xlsm`, the pipeline selects direct OOXML core + extraction and the OOXML rich backend by default. They share one ZIP through + final model construction. +3. If any uncaught failure or unsupported construct occurs in a direct OOXML + stage, the pipeline closes the session, discards every partial OOXML result, + and restarts the complete extraction through the openpyxl compatibility + pipeline. It logs `FallbackReason.OOXML_COMPATIBILITY` + (`ooxml_compatibility`) as a warning. +4. A drawing error isolated to one sheet retains the existing best-effort + behavior: omit the affected drawing artifacts for that sheet and continue + with healthy sheets. +5. An explicit `colors_map` request or an active legacy pipeline/helper/ + workbook override selects the complete openpyxl compatibility pipeline. +6. `libreoffice` retains its OOXML rich baseline and optional LibreOffice + enrichment; `standard` / `verbose` retain their COM behavior. `.xls` + selection and behavior are unchanged. +7. When COM succeeds, `colors_map` is overwritten with COM results. ## Workbook Resource Lifetime -- One extraction invocation owns an `OpenpyxlExtractionSession`. Compatible +- A supported `light` OOXML extraction owns one `OoxmlExtractionSession` and + one ZIP archive shared by core workbook parsing, drawing/chart extraction, + table candidates and final model construction. The session closes in a + `finally` path on success and failure before any compatibility restart. +- The openpyxl compatibility pipeline owns an `OpenpyxlExtractionSession`. + Compatible OOXML stages share one regular `data_only=True`, `read_only=False` workbook, including hyperlink and per-sheet table detection, until final modeling ends. - An openpyxl formula map opens a second regular `data_only=False` workbook lazily. Disabled formula extraction and COM-only `.xls` formula extraction must not open that variant. -- Without helper overrides, an OOXML extraction loads one openpyxl workbook, or - two when openpyxl formula extraction is enabled, independent of sheet count. - OOXML drawing ZIP reads are separate and are not included in that loader count. +- The normal supported `light` OOXML path opens one ZIP and no openpyxl + workbook. If direct extraction fails, the ZIP is closed and the full + compatibility pipeline starts from the beginning; results from the failed + attempt are not merged. - Standalone path-based helper calls own and close their own workbooks. An extraction does not cache workbooks across invocations or share them with another concurrent extraction. -- Success, fallback and exceptions release all successfully opened variants. - Existing helper overrides remain observable and may intentionally replace - the optimized session path. +- Success, fallback and exceptions release all successfully opened resources. + Existing helper and pipeline overrides remain observable and select the + compatibility path for `light` OOXML inputs. -## Internal OOXML core alternative +## OOXML core extraction session -Issue #149 adds a streaming `OoxmlExtractionSession` for core worksheet data -and optional rich extraction through one ZIP. It is available for parity and -future migration work; the pipeline above continues to use openpyxl. +Issue #149 added the streaming `OoxmlExtractionSession` for core worksheet +data. Issue #150 makes it the default core reader for supported `light` +`.xlsx` / `.xlsm` extraction, with rich extraction sharing the same ZIP. See [Core OOXML extraction session](ooxml-core-extraction.md) for supported values, formulas, relationships, tables, lifecycle and limitations. @@ -45,7 +62,10 @@ values, formulas, relationships, tables, lifecycle and limitations. ## Modes -- light: Skip COM entirely; return cells+table_candidates as the base, along with pre-com artifacts according to their flags, and on `.xlsx/.xlsm` emit best-effort OOXML shapes/connectors/charts when present +- light: Skip COM. Supported `.xlsx` / `.xlsm` files use the direct OOXML + pipeline for cells and pre-com artifacts plus best-effort shapes, connectors + and charts. Unsupported options or direct-extraction failures restart the + complete openpyxl compatibility pipeline. `.xls` behavior is unchanged. - libreoffice: Seed the same OOXML rich baseline as `light`, then try LibreOffice enrichment; on failure, fall back to cells with pre-com artifacts preserved and keep any OOXML rich artifacts already extracted - standard: Existing behavior (text-bearing shapes, charts if needed) - verbose: All shapes + sizes, charts with sizes @@ -53,22 +73,25 @@ values, formulas, relationships, tables, lifecycle and limitations. ## Import Boundaries - Bare package/engine imports and CLI help keep their existing lazy contracts. - Real `.xlsx/.xlsm` light extraction loads neither xlwings nor concrete COM - backends, COM shape/chart modules, ExStruct rendering or PDFium. + The normal supported `.xlsx` / `.xlsm` light path imports none of openpyxl, + pandas, SciPy or xlwings, nor concrete COM backends, ExStruct rendering or + PDFium. Unsupported input, `colors_map` opt-in or legacy overrides may select + the openpyxl compatibility path. - Shared cell/map/merged-range types do not import concrete extraction backends. Compatibility exports and override call sites resolve the live implementation only when requested or executed. -- Openpyxl and NumPy remain available when extraction needs them. Openpyxl may - import optional Pillow itself when installed; light extraction must also work - without Pillow. This transitive import does not activate ExStruct rendering. -- BIFF xlrd is imported only for `.xls` cell reading. SciPy labeling is imported - only when accelerated clustering is attempted; explicit Python clustering - does not import SciPy. +- Openpyxl remains installed for compatibility fallback and paths that select + it. NumPy remains required by the existing table-clustering contract. +- BIFF xlrd is imported only for `.xls` cell reading. Direct light OOXML table + heuristics explicitly select Python BFS per call and do not import SciPy. + Other modes and the openpyxl compatibility pipeline retain the configured + `EXSTRUCT_BORDER_CLUSTER_BACKEND` behavior. ## Cell Extraction -- Read `.xlsx` / `.xlsm` cached cell values directly with openpyxl and `.xls` - cached values directly with xlrd; no production pandas reader is used. +- Read supported `.xlsx` / `.xlsm` light-mode cached cell values directly from + OOXML. The compatibility pipeline retains openpyxl reading; `.xls` behavior + remains unchanged. No production pandas reader is used. - Ignore blank cells - Normalize row data into `CellRow` - Preserve existing numeric coercion (including zero-prefixed numeric strings), boolean/date/time text, @@ -85,7 +108,10 @@ values, formulas, relationships, tables, lifecycle and limitations. ## Table Extraction -- Merge openpyxl table definitions + border clusters +- In light OOXML, combine explicit table definitions with the existing + border-based table-candidate heuristics; preserve candidate output semantics. +- The openpyxl compatibility path retains openpyxl table definitions and border + clusters. - Preserve table_candidates even when COM is unavailable - Path-based detection delegates to worksheet-based detection. Nested border scanning and multi-sheet detection must not reopen the workbook when the @@ -94,10 +120,13 @@ values, formulas, relationships, tables, lifecycle and limitations. path matches the COM workbook path. A different workbook uses standalone path-based detection, even when both workbooks contain the same sheet name. - NumPy remains required. SciPy is optional through `exstruct[fast]` or `[all]`. - `EXSTRUCT_BORDER_CLUSTER_BACKEND=auto` attempts SciPy-backed labeling, while - `python` forces the existing Python BFS. The legacy `numpy` value also attempts - SciPy; import or execution failure falls back to Python. Unknown values retain - the existing `auto` behavior. Sparse-set BFS remains an evaluation candidate. + Direct OOXML light table heuristics pass `cluster_backend="python"` per call, + without reading or changing the environment variable. Other modes and the + openpyxl compatibility path retain `EXSTRUCT_BORDER_CLUSTER_BACKEND`: + `auto` attempts SciPy-backed labeling, `python` forces the existing Python + BFS, and legacy `numpy` also attempts SciPy. Import or execution failure falls + back to Python; unknown values retain existing `auto` behavior. Sparse-set BFS + remains an evaluation candidate. ## Shapes / Arrows / SmartArt Extraction @@ -129,7 +158,8 @@ Mode notes: ## Print Areas / Auto Page Breaks -- print_areas are retrieved via pre-com (openpyxl); COM only supplements missing parts +- Light OOXML reads print areas from workbook defined names. The compatibility + path retains openpyxl extraction; COM supplements missing parts as before. - auto_page_breaks are retrieved via COM only - The extraction CLI always exposes auto page-break export syntax and validates the required mode/runtime at execution time instead of probing COM during parser construction @@ -138,8 +168,21 @@ Mode notes: - Prefer COM to include conditional formatting colors - Overwrite with COM results when COM succeeds - Use openpyxl results only when COM fails +- An explicit `colors_map` request on the direct light OOXML path selects the + complete openpyxl compatibility pipeline. ## Error Handling / Fallback -- When COM / LibreOffice is unavailable or raises an exception, return cells+table_candidates, preserve the pre-com artifacts (print_areas / formulas_map / colors_map / merged_cells) according to their flags, and keep any OOXML rich artifacts that were already extracted before the failing enrichment step -- Log fallback reasons uniformly via `FallbackReason` +- Direct light OOXML failure or an unsupported construct at any pipeline stage + discards all partial OOXML data, closes the session, and restarts the complete + openpyxl compatibility pipeline. The warning and state use + `FallbackReason.OOXML_COMPATIBILITY = "ooxml_compatibility"` after a successful + compatibility restart. If compatibility extraction itself takes an existing + fallback, its later reason remains in state; the initial OOXML warning is + still logged. +- A per-sheet drawing parse failure remains best-effort and skips affected + drawing artifacts for that sheet while preserving healthy sheet output. +- COM / LibreOffice runtime failures retain their existing fallback behavior: + keep safe cell/table and pre-com artifacts, plus rich OOXML artifacts already + recovered where the current mode contract permits. +- Log fallback reasons uniformly via `FallbackReason`. diff --git a/dev-docs/specs/extraction-performance.md b/dev-docs/specs/extraction-performance.md index 5e5cd64e..b04e7102 100644 --- a/dev-docs/specs/extraction-performance.md +++ b/dev-docs/specs/extraction-performance.md @@ -109,6 +109,14 @@ checks against its own uninstrumented equivalent. Compare profiles with identica `profile.all_features` settings. Profile times include wrapper overhead and are for attribution, not substitutes for the uninstrumented latency observations. +For profile measurements, StageRecorder.install enables direct OOXML +instrumentation only for the normal light profile, with direct_ooxml true when +mode is light and all-features profiling is disabled. This instruments the +backend selected by the default request; it does not change backend selection. +For a direct OOXML profile, workbook_parsing reports zero calls because that +stage counts openpyxl load_workbook calls. OOXML XML parsing time is included in +the cells stage duration. + ## Fixtures and comparison `benchmark.performance_fixtures` generates six fixed synthetic categories; see @@ -141,3 +149,36 @@ light improvements from variable Excel COM observations and retains supplemental paired rechecks. `benchmark/issue148-results.md` records the separate accelerator comparison and ADR-0012 dependency decision; the candidate sparse BFS remains evaluation-only. + +## Issue #150 light-backend comparison + +benchmark/issue150_light.py compares the preserved openpyxl light pipeline with +the direct OOXML pipeline in separate worker processes. It records import and +extraction timings, cold in-process duration, repeated-extraction median, peak +RSS, workbook and ZIP open counts, imported heavy modules, fallback reason, and +serialized output hash. The runner stops if the two output hashes differ. + +The final Windows result file is +benchmark/baselines/issue150-2026-10-03-windows.json, recorded on Windows +10.0.22631 with Python 3.11.13. The seven inputs all produced matching +serialized-output hashes. OOXML used one ZIP archive, opened zero openpyxl +workbooks, and loaded none of openpyxl, pandas, SciPy or xlwings. The previous +pipeline opened two ZIP archives, one openpyxl workbook, and loaded openpyxl and +SciPy. See the [Issue #150 benchmark report](../../benchmark/issue150-results.md) +for the run details. + +| Input | openpyxl cold / warm median (ms) | OOXML cold / warm median (ms) | openpyxl / OOXML peak RSS (MiB) | +| --- | ---: | ---: | ---: | +| small.xlsx | 619.9 / 7.6 | 267.9 / 3.9 | 64.4 / 39.4 | +| large.xlsx | 955.1 / 481.1 | 645.5 / 407.2 | 109.3 / 71.8 | +| many-sheet.xlsx | 633.2 / 82.6 | 297.6 / 63.4 | 74.5 / 42.6 | +| sparse.xlsx | 644.9 / 39.3 | 256.9 / 15.9 | 74.5 / 39.5 | +| style-heavy.xlsx | 715.4 / 117.5 | 318.8 / 82.6 | 73.4 / 46.0 | +| table-heavy.xlsx | 616.9 / 57.4 | 281.0 / 50.1 | 75.3 / 44.1 | +| sample-shape-connector.xlsx | 588.7 / 13.0 | 244.9 / 5.5 | 64.5 / 39.6 | + +OOXML had a lower cold in-process duration, warm median and peak RSS on all +seven inputs in this run. The cold in-process metric includes module imports and +the first extraction, but excludes worker process startup and teardown measured +separately. These are observations from one Windows machine and one recorded +input set, not universal runtime guarantees. diff --git a/dev-docs/specs/ooxml-core-extraction.md b/dev-docs/specs/ooxml-core-extraction.md index 1a47d316..2d9460ce 100644 --- a/dev-docs/specs/ooxml-core-extraction.md +++ b/dev-docs/specs/ooxml-core-extraction.md @@ -4,9 +4,12 @@ Issue: [#149](https://github.com/harumiWeb/exstruct/issues/149) ## Scope and selection -`exstruct.core.ooxml_session.OoxmlExtractionSession` is an internal, read-only -alternative for `.xlsx` / `.xlsm` core extraction. It does not replace the -openpyxl pipeline or change public models. Default-path migration is separate. +`exstruct.core.ooxml_session.OoxmlExtractionSession` is the read-only core +reader for the normal `.xlsx` / `.xlsm` `light` path. It shares one ZIP with +the OOXML rich backend and does not change public models. The openpyxl pipeline +remains the complete compatibility path and is selected for unsupported +constructs, direct-stage failures, `colors_map` opt-in, or active legacy +pipeline/helper/workbook overrides. See ADR-0013. ```python from pathlib import Path @@ -41,9 +44,10 @@ with OoxmlExtractionSession(path) as session: Existing drawing parsing retains its separate streaming geometry scan. - Closing releases the ZIP and caches, including after context-body errors. Accessing a closed session raises RuntimeError. -- No openpyxl workbook/worksheet model is created. Existing value/formula - normalization and openpyxl scalar format, date and formula-translation helpers - are reused; openpyxl remains a dependency. +- No openpyxl workbook/worksheet model is created. Pure-Python scalar format, + date and formula-translation helpers preserve the existing value contract. + The normal supported `light` path does not import openpyxl, pandas, SciPy or + xlwings; openpyxl remains installed for compatibility and other paths. - Core malformed XML, missing required parts and invalid scalar data raise errors. Missing optional relationship parts return an empty mapping. No partially parsed worksheet is cached. Rich extraction retains its existing @@ -73,8 +77,15 @@ with OoxmlExtractionSession(path) as session: groups normalized `=`-prefixed formulas at `(row, zero-based column)` coordinates. Shared followers translate relative references from the shared anchor. Array/dynamic formulas retain explicit anchor text without inventing follower - formulas. Unresolved shared followers are omitted; unsupported translation - is logged and skipped. + formulas. Unresolved shared followers are omitted. Shared translation of + whole-row/column ranges, unquoted sheet-qualified references, non-ASCII names, + or references crossing worksheet bounds raises `UnsupportedOoxmlError`; + the public light pipeline restarts compatibility extraction instead of + emitting a partially translated formula map. Quoted sheet names and string + literals are preserved by the ordinary A1 translator. +- Date/duration style classification is cached once per session. Elapsed-time + formats are case-insensitive, including `[H]:MM:SS`, and preserve both row + display text and raw merged-anchor values. - External hyperlink URLs are preserved exactly. Internal `location` links are omitted, matching current output. Links on filtered values survive only when their row has another emitted value. Range links use zero-based column keys. @@ -85,6 +96,28 @@ with OoxmlExtractionSession(path) as session: and skipped. - `extract_explicit_tables()` returns referenced table names, display names, ranges and column names, without the border heuristics of `table_candidates`. +- `detect_tables(name, mode="light")` also applies the existing border-based + heuristics to produce `table_candidates` on the direct OOXML path. It passes + `cluster_backend="python"` per call so the normal path avoids importing SciPy + without reading or changing `EXSTRUCT_BORDER_CLUSTER_BACKEND`. + +## Pipeline selection and recovery + +- The normal `light` pipeline uses one session archive for core worksheet data, + formulas, hyperlinks, merged cells, print areas, table candidates, shapes and + charts through final model construction. +- Any uncaught error or unsupported construct in any direct OOXML stage + discards all partial OOXML results. The session closes in a `finally` path, + then extraction restarts from the beginning through the complete openpyxl + compatibility pipeline. +- The warning and extraction state use + `FallbackReason.OOXML_COMPATIBILITY` (`ooxml_compatibility`). Known + unsupported options and legacy overrides select compatibility before the + ZIP is opened where possible. +- Existing per-sheet drawing resilience remains: a drawing failure isolated to + one sheet may omit that sheet's affected drawing artifacts while preserving + core data and healthy-sheet drawings. +- Other modes and `.xls` selection/behavior remain unchanged. ## Relationships @@ -120,18 +153,12 @@ parity, Ruff, strict mypy and diff checks passed. These fixes restore extraction semantics and improve internal indexing; they introduce no backend-selection, public mode or fallback policy change and require no new ADR. -## ADR assessment - -- Verdict: `not-needed`; next action: `no-adr`. -- Rationale: internal parsing capabilities are added without changing backend - selection, priority, mode meanings, public output or fallback policy. A future - pipeline migration requires a fresh ADR assessment. -- Domains: extraction implementation and resource reuse. -- Existing decisions: ADR-0010 (light OOXML rich baseline), ADR-0002 (rich fallback). -- Evidence triad: this spec and `dev-docs/specs/excel-extraction.md`; - `src/exstruct/core/ooxml_session.py`, `ooxml_package.py`, `ooxml_drawing.py`, - `backends/ooxml_backend.py`; `tests/core/test_ooxml_extraction_session.py`, - `test_ooxml_package.py`, `test_ooxml_drawing.py` and existing pipeline tests. +## Decision history + +Issue #149 added reusable OOXML parsing primitives without changing backend +selection; its original no-ADR assessment applied to that capability-only +scope. Issue #150 changes the default backend and whole-pipeline fallback +contract. ADR-0013 records that policy and applies to this specification. ## Validation diff --git a/dev-docs/testing/test-requirements.md b/dev-docs/testing/test-requirements.md index b172f56f..20d5ef85 100644 --- a/dev-docs/testing/test-requirements.md +++ b/dev-docs/testing/test-requirements.md @@ -215,7 +215,8 @@ Overall code coverage must be **80% or higher**. - [INT-02] Preserve print_areas even during COM fallback - [IO-05] `dict_without_empty_values` removes None/empty list/empty dict and preserves nesting - [RENDER-01] PDF/PNG smoke tests for Excel+COM+pypdfium2 (env ON/OFF) -- [MODE-08] In light, extract print_areas with openpyxl and keep them in default output unless include_print_areas=False is explicitly requested +- [MODE-08] In light, include print_areas by default unless include_print_areas=False is explicitly requested; supported .xlsx / .xlsm reads them from OOXML +- [MODE-09] Normal supported .xlsx / .xlsm light extraction imports none of openpyxl, pandas, SciPy, or xlwings and shares one ZIP between core and rich extraction ## 9.1 Pipeline @@ -227,7 +228,10 @@ Overall code coverage must be **80% or higher**. - [PIPE-06] print_areas preserves openpyxl results, and COM supplements only missing parts - [PIPE-07] PipelineState holds com_attempted/com_succeeded/fallback_reason - [PIPE-08] Do not include the COM step for auto_page_breaks when include_auto_page_breaks=False -- [PIPE-09] Do not include the extraction step for merged_cells when include_merged_cells=False +- [PIPE-09] Do not include the extraction step for merged_cells when include_merged_cells=False +- [PIPE-10] An uncaught failure or unsupported construct in any direct light OOXML stage discards partial data, closes the session, and restarts the complete openpyxl pipeline with `ooxml_compatibility` +- [PIPE-11] colors_map opt-in and active legacy helper/pipeline/workbook overrides select complete compatibility before OOXML ZIP access where detectable +- [PIPE-12] Direct light OOXML table heuristics pass `cluster_backend="python"` per call without changing `EXSTRUCT_BORDER_CLUSTER_BACKEND`; other modes and compatibility preserve ADR-0012 selection - [PIPE-MOD-01] build_workbook_data builds WorkbookData/SheetData from raw containers - [PIPE-MOD-02] collect_sheet_raw_data collects extracted data into raw containers diff --git a/docs/api.md b/docs/api.md index bafa6aa5..a5931d27 100644 --- a/docs/api.md +++ b/docs/api.md @@ -83,7 +83,13 @@ process_excel( ## Extraction Mode Guide -- `mode="light"` is the pure-Python baseline. It skips COM entirely and, for `.xlsx/.xlsm`, returns best-effort OOXML shapes, connectors, and charts in addition to cells, table candidates, and print areas. +- `mode="light"` is the pure-Python baseline. For `.xlsx` / `.xlsm` it reads + core data and best-effort shapes, connectors, and charts through one shared + OOXML archive. The normal path does not import openpyxl, pandas, SciPy, or + xlwings. Unsupported OOXML, `colors_map` opt-in, or an active legacy helper + override restarts the complete openpyxl compatibility pipeline and logs the + `ooxml_compatibility` warning reason; partial OOXML results are discarded. + Per-sheet drawing failures retain the existing best-effort behavior. - `mode="libreoffice"` starts from the same OOXML baseline as `light` and then applies optional LibreOffice enrichment when the runtime is available. - `mode="standard"` and `mode="verbose"` remain the COM-backed paths when you need native Excel fidelity. @@ -147,13 +153,17 @@ the MCP server, which wraps the same core and adds host policy. ## Dependencies -- Core cell extraction: openpyxl for `.xlsx` / `.xlsm`, xlrd for `.xls` - (installed with the package). Production extraction does not require pandas. +- Core cell extraction: the normal `light` path reads `.xlsx` / `.xlsm` directly + from OOXML. Openpyxl remains installed for compatibility fallback and other + paths; xlrd reads `.xls` (installed with the package). Production extraction + does not require pandas. - Border clustering: NumPy is required; SciPy is optional via `exstruct[fast]` (also included in `exstruct[all]`). `EXSTRUCT_BORDER_CLUSTER_BACKEND=auto` prefers SciPy when available and falls back to the existing Python BFS on import or execution failure. `python` forces BFS; the legacy `numpy` name - selects SciPy-backed labeling with the same fallback. No `scipy` alias is added. + selects SciPy-backed labeling with the same fallback. Direct light OOXML + table detection uses Python BFS regardless of this setting; other modes and the + compatibility path retain the configured selection. No `scipy` alias is added. - YAML export: `pyyaml` (lazy import; missing module raises `MissingDependencyError`). - TOON export: `python-toon` (lazy import; missing module raises `MissingDependencyError`). - Auto page-break extraction/export: **Excel + COM** required. `mode="libreoffice"` rejects auto page-break requests with `ConfigError`. diff --git a/docs/cli.md b/docs/cli.md index 1269da10..bf475db4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -111,7 +111,7 @@ empty workbook. | ---- | ----------- | | `-o, --output PATH` | Output path. Omit to write to stdout. | | `-f, --format {json,yaml,yml,toon}` | Serialization format (default: `json`). | -| `-m, --mode {light,libreoffice,standard,verbose}` | Extraction detail level.
- light: pure-Python baseline; returns cells + table candidates + print areas, and on `.xlsx/.xlsm` adds best-effort OOXML shapes, connectors, and charts.
- libreoffice: optional non-COM enrichment mode for `.xlsx/.xlsm`; starts from the same OOXML baseline as `light` and uses LibreOffice runtime to refine merged cells, shapes, connectors, and charts when available.
- standard: shapes with text/arrows + charts + print areas via Excel COM.
- verbose: all shapes/charts with size + hyperlinks/maps via Excel COM. | +| `-m, --mode {light,libreoffice,standard,verbose}` | Extraction detail level.
- light: for `.xlsx` / `.xlsm`, reads core data and best-effort shapes/connectors/charts through one OOXML archive; other input behavior is unchanged.
- libreoffice: optional non-COM enrichment mode for `.xlsx/.xlsm`; starts from the same OOXML baseline as `light` and uses LibreOffice runtime to refine merged cells, shapes, connectors, and charts when available.
- standard: shapes with text/arrows + charts + print areas via Excel COM.
- verbose: all shapes/charts with size + hyperlinks/maps via Excel COM. | | `--alpha-col` | Output column keys as Excel-style names (`A`, `B`, ..., `AA`) instead of 0-based numeric keys (`"0"`, `"1"`, ...). Default: disabled (legacy numeric keys). | | `--pretty` | Pretty-print JSON (indent=2). | | `--image` | Render per-sheet PNGs (requires Excel + COM + `pypdfium2`; not supported in `--mode libreoffice`). | @@ -157,6 +157,8 @@ exstruct sample.xlsx --pdf --image --dpi 144 -o out.json safety policy. For direct Python workbook editing, `openpyxl` / `xlwings` are usually the better fit. - On non-COM environments, prefer `--mode light` as the pure-Python baseline for `.xlsx/.xlsm`; use `--mode libreoffice` only when you want optional LibreOffice-based enrichment on top of that baseline. +- The normal supported light path does not import openpyxl, pandas, SciPy, or xlwings. Unsupported OOXML or `colors_map` requests discard partial direct results, restart the complete openpyxl compatibility pipeline, and log the `ooxml_compatibility` warning reason. A drawing failure isolated to one sheet remains best-effort. +- Direct light OOXML table heuristics select Python BFS per call without changing `EXSTRUCT_BORDER_CLUSTER_BACKEND`; other modes and compatibility extraction retain the configured clustering selection. - `--mode libreoffice` is best-effort, not a strict subset of COM output. It does not render PDFs/PNGs and does not compute auto page-break areas in v1. - `--auto-page-breaks-dir` is always shown in help output and is validated at execution time. - `--mode libreoffice` combined with `--pdf`, `--image`, or `--auto-page-breaks-dir` fails early with a configuration error instead of silently ignoring the option. diff --git a/src/exstruct/core/backends/ooxml_backend.py b/src/exstruct/core/backends/ooxml_backend.py index 1e65d053..86c52d5b 100644 --- a/src/exstruct/core/backends/ooxml_backend.py +++ b/src/exstruct/core/backends/ooxml_backend.py @@ -10,7 +10,7 @@ from defusedxml import ElementTree from ...models import Chart -from ..ooxml_drawing import SheetDrawingData, read_sheet_drawings +from ..ooxml_drawing import SheetDrawingData, read_sheet_drawings as read_sheet_drawings from .base import ChartData, RichBackend, ShapeData from .ooxml_shapes import _build_shapes_from_ooxml @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) _OOXML_SUFFIXES = {".xlsx", ".xlsm"} +_DEFAULT_READ_SHEET_DRAWINGS = read_sheet_drawings class OoxmlRichBackend(RichBackend): diff --git a/src/exstruct/core/backends/openpyxl_backend.py b/src/exstruct/core/backends/openpyxl_backend.py index 2d44c070..ee758a98 100644 --- a/src/exstruct/core/backends/openpyxl_backend.py +++ b/src/exstruct/core/backends/openpyxl_backend.py @@ -23,7 +23,7 @@ ) from ..openpyxl_session import OpenpyxlExtractionSession from ..ranges import parse_range_zero_based -from ..workbook import openpyxl_workbook +from ..workbook import openpyxl_workbook as openpyxl_workbook from .base import CellData, MergedCellData, PrintAreaData logger = logging.getLogger(__name__) diff --git a/src/exstruct/core/cells.py b/src/exstruct/core/cells.py index 7188cd79..457024b1 100644 --- a/src/exstruct/core/cells.py +++ b/src/exstruct/core/cells.py @@ -16,12 +16,6 @@ from typing import TYPE_CHECKING, Literal import numpy as np -from openpyxl.cell.cell import Cell, MergedCell -from openpyxl.cell.read_only import EmptyCell, ReadOnlyCell -from openpyxl.styles.colors import Color -from openpyxl.utils import get_column_letter, range_boundaries -from openpyxl.worksheet._read_only import ReadOnlyWorksheet -from openpyxl.worksheet.worksheet import Worksheet from ..models import CellRow from ._lazy import LazyModule @@ -36,6 +30,11 @@ from .workbook import openpyxl_workbook if TYPE_CHECKING: + from openpyxl.cell.cell import Cell, MergedCell + from openpyxl.cell.read_only import EmptyCell, ReadOnlyCell + from openpyxl.styles.colors import Color + from openpyxl.worksheet._read_only import ReadOnlyWorksheet + from openpyxl.worksheet.worksheet import Worksheet import xlwings as xw else: xw = LazyModule("xlwings") @@ -65,6 +64,23 @@ ExtractionMode = Literal["light", "libreoffice", "standard", "verbose"] +def get_column_letter(index: int) -> str: + """Resolve scalar coordinates without importing compatibility backends.""" + from .ooxml_scalars import get_column_letter as column_letter + + return column_letter(index) + + +def range_boundaries(ref: str) -> tuple[int, int, int, int]: + """Resolve finite table/cell extents without importing openpyxl.""" + from .ooxml_scalars import range_boundaries as bounds + + c1, r1, c2, r2 = bounds(ref) + if c1 is None or r1 is None or c2 is None or r2 is None: + raise ValueError(f"Non-finite cell range: {ref}") + return c1, r1, c2, r2 + + @dataclass(frozen=True) class TableScanLimits: """Limits for openpyxl border scanning during table detection.""" @@ -712,6 +728,8 @@ def extract_sheet_cells_openpyxl_ws( retained when their row has another emitted value; link-only rows are not invented. Worksheet ownership and closing remain with the caller. """ + from openpyxl.worksheet.worksheet import Worksheet + cell_rows: Iterable[ tuple[int, Iterable[tuple[int, Cell | MergedCell | ReadOnlyCell | EmptyCell]]] ] @@ -1247,7 +1265,10 @@ def _resolve_border_cluster_backend() -> Literal["auto", "python", "numpy"]: def detect_border_clusters( - has_border: np.ndarray, min_size: int = 4 + has_border: np.ndarray, + min_size: int = 4, + *, + backend: Literal["auto", "python", "numpy"] | None = None, ) -> list[tuple[int, int, int, int]]: """Detect border clusters using the selected backend. @@ -1258,7 +1279,7 @@ def detect_border_clusters( Returns: List of bounding boxes (r1, c1, r2, c2). """ - backend = _resolve_border_cluster_backend() + backend = backend or _resolve_border_cluster_backend() if backend == "python": return _detect_border_clusters_python(has_border, min_size) try: @@ -1772,7 +1793,10 @@ def cell_has_any_border(r: int, c: int) -> bool: def _detect_border_rectangles( - has_border: np.ndarray | Sequence[Sequence[bool]], *, min_size: int + has_border: np.ndarray | Sequence[Sequence[bool]], + *, + min_size: int, + cluster_backend: Literal["auto", "python", "numpy"] | None = None, ) -> list[tuple[int, int, int, int]]: """Detect border rectangles from a boolean grid. @@ -1783,7 +1807,10 @@ def _detect_border_rectangles( Returns: List of rectangles as (top_row, left_col, bottom_row, right_col). """ - return detect_border_clusters(np.asarray(has_border, dtype=bool), min_size=min_size) + grid = np.asarray(has_border, dtype=bool) + if cluster_backend is None: + return detect_border_clusters(grid, min_size=min_size) + return detect_border_clusters(grid, min_size=min_size, backend=cluster_backend) def _merge_rectangles( @@ -1979,6 +2006,7 @@ def detect_tables_openpyxl_ws( *, mode: ExtractionMode = "standard", scan_limits: TableScanLimits | None = None, + cluster_backend: Literal["auto", "python", "numpy"] | None = None, ) -> list[str]: """Detect tables using an already loaded worksheet.""" resolved_limits = _resolve_table_scan_limits(mode, scan_limits) @@ -1990,7 +2018,13 @@ def detect_tables_openpyxl_ws( scan_limits=resolved_limits, ) ) - rects = _detect_border_rectangles(has_border, min_size=4) + rects = ( + _detect_border_rectangles(has_border, min_size=4) + if cluster_backend is None + else _detect_border_rectangles( + has_border, min_size=4, cluster_backend=cluster_backend + ) + ) merged_rects = _merge_rectangles(rects) dedup: set[str] = set(tables) diff --git a/src/exstruct/core/ooxml_scalars.py b/src/exstruct/core/ooxml_scalars.py new file mode 100644 index 00000000..082d49de --- /dev/null +++ b/src/exstruct/core/ooxml_scalars.py @@ -0,0 +1,273 @@ +"""Pure-Python scalar helpers for reading common OOXML workbook values.""" + +from __future__ import annotations + +from datetime import date, datetime, time, timedelta as duration +import re + + +class UnsupportedOoxmlError(Exception): + """Signal that an OOXML construct needs the compatibility backend.""" + + +_MAX_COLUMN = 18278 # openpyxl's accepted three-letter column boundary +_CELL_RE = re.compile(r"^\$?([A-Za-z]{1,3})\$?([1-9][0-9]*)$") +_COLUMN_RE = re.compile(r"^\$?([A-Za-z]{1,3})$") +_ROW_RE = re.compile(r"^\$?([1-9][0-9]*)$") +_FORMULA_CELL_RE = re.compile( + r"(?(?P\d{4})-(?P\d{2})-(?P\d{2}))?T?" + r"(?P