diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23f980d8..af437c51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,13 @@ jobs: if: env.RUN == 'true' && matrix.os == 'ubuntu-latest' run: dotnet format whitespace ExcelReader.slnx --verify-no-changes + # Packing runs package validation, which fails on any binary break against the last release. + - name: Validate packages against the last release + if: env.RUN == 'true' && matrix.os == 'ubuntu-latest' + run: | + dotnet pack src/ExcelReader.Core/ExcelReader.Core.csproj --configuration Release --no-build --output ./package-validation + dotnet pack src/ExcelReader.Arrow/ExcelReader.Arrow.csproj --configuration Release --no-build --output ./package-validation + - name: Test (Release) if: env.RUN == 'true' run: >- diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a0b08d88..269c03d5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,7 +24,8 @@ jobs: if: github.event.pull_request.draft != true runs-on: ubuntu-latest outputs: - languages: ${{ steps.languages.outputs.value }} + csharp: ${{ github.event_name != 'pull_request' || steps.filter.outputs.csharp == 'true' }} + python: ${{ github.event_name != 'pull_request' || steps.filter.outputs.python == 'true' }} steps: - name: Checkout uses: actions/checkout@v7 @@ -45,39 +46,32 @@ jobs: - 'python/**' - '.github/workflows/codeql.yml' - - name: Select languages - id: languages - run: | - if [ '${{ github.event_name }}' != 'pull_request' ]; then - echo 'value=["csharp","python"]' >> "$GITHUB_OUTPUT" - exit 0 - fi - languages=() - if [ '${{ steps.filter.outputs.csharp }}' = 'true' ]; then - languages+=('"csharp"') - fi - if [ '${{ steps.filter.outputs.python }}' = 'true' ]; then - languages+=('"python"') - fi - echo "value=[$(IFS=,; echo "${languages[*]}")]" >> "$GITHUB_OUTPUT" - + # Both languages always get a job: "Analyze (csharp)" and "Analyze (python)" are required checks, and + # a check whose job never starts stays pending. An unchanged language's job skips its steps instead. analyze: name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest needs: changes - if: needs.changes.outputs.languages != '[]' strategy: fail-fast: false matrix: - language: ${{ fromJSON(needs.changes.outputs.languages) }} + language: [csharp, python] + + env: + RUN: ${{ needs.changes.outputs[matrix.language] }} steps: + - name: Skip (language unchanged) + if: env.RUN != 'true' + run: echo "No ${{ matrix.language }} changes; skipping the analysis." + - name: Checkout + if: env.RUN == 'true' uses: actions/checkout@v7 - name: Setup .NET - if: matrix.language == 'csharp' + if: env.RUN == 'true' && matrix.language == 'csharp' uses: actions/setup-dotnet@v6 with: global-json-file: global.json @@ -85,6 +79,7 @@ jobs: cache-dependency-path: '**/*.csproj' - name: Initialize CodeQL + if: env.RUN == 'true' uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} @@ -93,13 +88,14 @@ jobs: # Only the shipped projects: tests, benchmarks and the fuzz harness double the build for code # no one runs against untrusted input. - name: Build (Release) - if: matrix.language == 'csharp' + if: env.RUN == 'true' && matrix.language == 'csharp' run: | for project in src/*/*.csproj; do dotnet build "$project" --configuration Release -p:RunAnalyzers=false done - name: Perform CodeQL Analysis + if: env.RUN == 'true' uses: github/codeql-action/analyze@v4 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 06d6862d..527b73a1 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -62,7 +62,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -172,7 +172,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -202,6 +202,12 @@ jobs: - name: Run the Python tests run: pytest python/tests -v + - name: Build the wheel and check it installs here + run: | + pip install build + python -m build --wheel python/ --outdir dist/ + python python/scripts/check_wheel_tag.py dist + - name: Build the C smoke test run: | cmake -S tests/ExcelReader.NativeSmoke -B build/nativesmoke -DCMAKE_BUILD_TYPE=Release @@ -215,6 +221,23 @@ jobs: python python/benchmarks/bench_read.py --n 1 python python/benchmarks/bench_write.py --n 1 + test-musl: + name: musl library + Python (${{ matrix.runner }}) + needs: changes + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.python == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build, test and wheel in Alpine + run: sh python/scripts/build_musl_wheel.sh + test-cpp: name: C++ package + smoke test (${{ matrix.os }}) needs: [changes, build-native] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91fff678..1f367a03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,7 +67,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -92,7 +92,7 @@ jobs: macOS) src=ExcelReader.Native.dylib; os=osx; ext=dylib ;; *) src=ExcelReader.Native.so; os=linux; ext=so ;; esac - arch=$([ "$(uname -m 2>/dev/null || echo x64)" = "arm64" ] && echo arm64 || echo x64) + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; *) arch=x64 ;; esac mkdir -p release-assets cp "python/src/excelreader/_lib/${src}" "release-assets/excelreader-native-${os}-${arch}.${ext}" @@ -116,7 +116,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -133,7 +133,7 @@ jobs: id: rid shell: bash run: | - arch=$([ "$(uname -m 2>/dev/null || echo x64)" = "arm64" ] && echo arm64 || echo x64) + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; *) arch=x64 ;; esac case "${{ runner.os }}" in Windows) os=win ;; macOS) os=osx ;; @@ -175,7 +175,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -209,6 +209,9 @@ jobs: pip install build python -m build --wheel python/ --outdir dist/ + - name: Check the wheel installs here + run: python python/scripts/check_wheel_tag.py dist + - name: Upload wheel artifact uses: actions/upload-artifact@v7 with: @@ -216,9 +219,48 @@ jobs: path: dist/*.whl if-no-files-found: error + musl: + name: Build musl library and wheel (${{ matrix.arch }}) + needs: build + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - { arch: x64, runner: ubuntu-latest } + - { arch: arm64, runner: ubuntu-24.04-arm } + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set package version + run: sed -i "s/^version = \".*\"/version = \"${{ needs.build.outputs.version }}\"/" python/pyproject.toml + + - name: Build, test and wheel in Alpine + run: sh python/scripts/build_musl_wheel.sh + + - name: Rename asset for this platform + run: | + mkdir -p release-assets + cp python/src/excelreader/_lib/ExcelReader.Native.so "release-assets/excelreader-native-linux-musl-${{ matrix.arch }}.so" + + - name: Upload release asset + uses: actions/upload-artifact@v7 + with: + name: release-native-musl-${{ matrix.arch }} + path: release-assets/* + if-no-files-found: error + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: wheel-musl-${{ matrix.arch }} + path: dist/*.whl + if-no-files-found: error + publish: name: Publish to NuGet & GitHub Release - needs: [build, native, cli, python] + needs: [build, native, cli, python, musl] runs-on: ubuntu-latest permissions: contents: write diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82fea814..b3150733 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,21 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas `src/ExcelReader.Core/PublicAPI/PublicAPI.Unshipped.txt`. A bot promotes `Unshipped` → `Shipped` automatically after each release — don't edit `Shipped.txt` by hand. +- **Minor versions don't break callers.** [docs/guide/versioning.md](docs/guide/versioning.md) is the + policy; in practice: + - A new member on a library-implemented interface (`IExcelWorkbook`, `IExcelSheet`, + `IExcelRowEnumerator`, `IWorkbookWriter`, `ISheetWriter`, `ITableSheetWriter`, `IRowWriter`) + gets a default implementation. A user-implemented one (`IExcelCellConverter`, `IExcelCellWriter`, + `IExcelRowMap`, `IExcelRecordMap`, `IRowRecord`, `IRowAccumulator`) gets no new member a user + would have to implement. + - A feature only some formats support goes on the concrete types first, or on its own interface + the way `ITableSheetWriter` sits beside `ISheetWriter`. + - New inputs go on the options object or a new overload, never as a new optional parameter on an + existing method. + - `dotnet pack` runs package validation against `PackageValidationBaselineVersion` in the + `ExcelReader.Core` and `ExcelReader.Arrow` projects, and CI runs it on every pull request. + After each release, move the baseline to the version just published. + - **Tests are required for behavior changes.** Run the suite before opening a PR: ```bash diff --git a/Directory.Build.props b/Directory.Build.props index 8782a287..0e6a30ac 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -23,11 +23,11 @@ it used to arrive transitively through Roslyn.Diagnostics.Analyzers, which brought a whole Roslyn-internal ruleset along for those three rules. --> - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/cpp/README.md b/cpp/README.md index d756c881..bbcf5bcc 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -2,8 +2,8 @@ # excelreader (C++) -Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path or -memory, with the full open-options surface), sheet navigation, schema inference, schema-driven +Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path, +memory, a random-access `xl::Source` or an `xl::InputStream`, with the full open-options surface), sheet navigation, schema inference, schema-driven typed table parsing, schema-driven writing, row-by-row decoded reads, and Arrow export. ## Requirements @@ -20,7 +20,7 @@ typed table parsing, schema-driven writing, row-by-row decoded reads, and Arrow include(FetchContent) FetchContent_Declare(excelreader GIT_REPOSITORY https://github.com/GabrielMarquezMatte/ExcelReader.git - GIT_TAG v4.0.1 + GIT_TAG v7.0.0 SOURCE_SUBDIR cpp) FetchContent_MakeAvailable(excelreader) @@ -47,14 +47,14 @@ cmake --install build/cpp --config Release --prefix /your/prefix ``` ```cmake -find_package(excelreader 3.0 REQUIRED) +find_package(excelreader 7.0 REQUIRED) target_link_libraries(your_app PRIVATE xl::excelreader) ``` Point `CMAKE_PREFIX_PATH` at the prefix you installed into. The native binary is resolved once, at install time — `find_package` never touches the network. The package version comes from `EXCELREADER_VERSION`, so a checkout that isn't on a tag installs as `0.0.0` and any versioned -`find_package` request against it fails; pass `-DEXCELREADER_VERSION=v3.0.2` to install under a real +`find_package` request against it fails; pass `-DEXCELREADER_VERSION=v7.0.0` to install under a real version. On Windows the generated import library is installed next to the DLL, and the package ships `excelreader_copy_native_library` as well, so a `find_package` consumer copies the DLL next to its executable the same way a `FetchContent` one does. @@ -93,14 +93,14 @@ template<> struct xl::ExcelMapper { }; auto workbook = xl::Workbook::open("book.xlsx"); -auto table = xl::parse_sheet(workbook->sheet(0)); +auto table = xl::parse(workbook->sheet(0)); for (const auto& row : *table) { /* ... */ } ``` ### Sheets and schema inference ```cpp -for (const auto& name : *workbook->sheet_names()) { /* ... */ } +for (xl::Sheet sheet : *workbook->sheets()) { std::println("{}", sheet.name().value()); } // Guess a schema from the header row plus a sample of the data, before committing to one. for (const auto& column : *workbook->sheet(1).infer_schema(1, 100)) { @@ -140,7 +140,7 @@ One open workbook can be read from several threads, which loads the shared-strin std::vector threads; for (xl::Sheet sheet : sheets) { - threads.emplace_back([sheet] { auto table = xl::parse_sheet(sheet); /* ... */ }); + threads.emplace_back([sheet] { auto table = xl::parse(sheet); /* ... */ }); } for (std::thread &thread : threads) { @@ -171,27 +171,27 @@ while (auto row = cursor.next_row()) ``` Each row is invalidated by the next `next_row()`. To hold every row at once, use -`read_all_decoded()`, which decodes the whole sheet in one native call and owns it until -destroyed (`RowCursor::read_all_decoded()` does the same for the rows a cursor has left): +`Sheet::read_all()`, which decodes the whole sheet in one native call and owns it until +destroyed: ```cpp -auto rows = workbook.sheet(0).read_all_decoded().value(); +auto rows = workbook.sheet(0).read_all().value(); for (auto row : rows) { std::println("{} cells", row.size()); } ``` -`xl::parse_sheet` remains the fastest way to read a sheet whose columns you know. +`xl::parse` remains the fastest way to read a sheet whose columns you know. ### Chunked reads -For a sheet too large to hold in memory at once, `xl::typed_reader` is `xl::parse_sheet` +For a sheet too large to hold in memory at once, `xl::parse_batches` is `xl::parse` delivered a batch at a time: ```cpp auto workbook = xl::Workbook::open("book.xlsx").value(); -auto reader = xl::typed_reader(workbook.sheet(0), 1, 10'000).value(); +auto reader = xl::parse_batches(workbook.sheet(0), {.batch_size = 10'000}).value(); for (auto &batch : reader) { if (!batch) { break; } @@ -199,24 +199,22 @@ for (auto &batch : reader) } ``` -`typed_reader` returns `std::expected, xl::Error>`; `next()` returns +`parse_batches` returns `std::expected, xl::Error>`; `next()` returns `std::expected>, Error>`, an empty optional at end of sheet, and `TypedReader` is also an input range, so the range-`for` above works directly. `batch_size` is rows -per batch — `0` means one unbounded batch, identical to `parse_sheet`; negative is an error. Each -batch is independent and outlives the reader. +per batch and must be positive (`0` or negative is `XL_INVALID_ARGUMENT`; read a whole sheet with +`parse`). Each batch is independent and outlives the reader. Several typed readers and Arrow streams may be open on one workbook at once; each read is independent. ### Encrypted workbooks -`OpenOptions::password(std::string_view)` unlocks a password-protected OOXML workbook -(.xlsx/.xlsb/.xlsm) through `Workbook::open`/`open_memory`: +`OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm) through +any `Workbook::open` overload: ```cpp -xl::OpenOptions options{}; -options.password("hunter2"); -auto workbook = xl::Workbook::open("protected.xlsx", XL_FORMAT_AUTO, &options); +auto workbook = xl::Workbook::open("protected.xlsx", {.password = "hunter2"}); if (!workbook) { if (workbook.error().code == XL_STATUS_PASSWORD_REQUIRED) { /* no password was supplied */ } else if (workbook.error().code == XL_STATUS_PASSWORD_INCORRECT) { /* ask again */ } @@ -226,26 +224,22 @@ if (!workbook) { Unlike the Rust/Python bindings, this header never throws — `XL_STATUS_PASSWORD_REQUIRED` and `XL_STATUS_PASSWORD_INCORRECT` (from `excelreader.h`) are reported as ordinary `xl::Error::code` -values on the `std::unexpected` returned from `open`/`open_memory`, distinguishable from any other -failure without parsing `error().message`. - -The `password_` string this builds into `xl_open_options::password` must outlive the open call; -keeping it in the `OpenOptions` object (which owns its own copy) is what makes -`options.password("hunter2")` safe to call with a temporary, as above. +values on the `std::unexpected` returned from `open`, distinguishable from any other failure without +parsing `error().message`. Writing an encrypted workbook is a second step: write the plaintext package with `write_columns`/ `write_sheet`, then wrap it with `xl::encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container — AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: ```cpp -auto written = xl::write_columns("plain.xlsx", XL_FORMAT_XLSX, columns); +auto written = xl::write_columns("plain.xlsx", columns); auto encrypted = xl::encrypt_package("plain.xlsx", "secret.xlsx", "hunter2"); if (!encrypted) { /* encrypted.error().message */ } ``` `package_path` is read twice (it is not disposed or removed), so it must already be a finished file. Encryption parameters are fixed at Excel's own defaults — there are no options — and only XLSX/XLSB -packages can be encrypted, matching what `Workbook::open`/`open_memory` can decrypt. +packages can be encrypted, matching what `Workbook::open` can decrypt. ### Reading from your own bytes @@ -268,7 +262,7 @@ public: } }; -auto workbook = xl::Workbook::open_stream(std::make_unique(), XL_FORMAT_CSV); +auto workbook = xl::Workbook::open(std::make_unique(), {.format = XL_FORMAT_CSV}); ``` The workbook owns the object you pass and deletes it once nothing needs it. A returned error @@ -285,16 +279,16 @@ Arrow implementation you already link. #include auto workbook = xl::Workbook::open("book.xlsx"); -auto table = xl::parse_arrow(workbook->sheet(0)); +auto table = xl::to_arrow(workbook->sheet(0)); // table->array / table->schema are a top-level struct array; both release in ~ArrowTable. ``` -Batched: `xl::arrow_stream` is `xl::parse_arrow` delivered a batch at a time. `schema()` and -`next()` return RAII guards (`ArrowSchemaGuard`, `ArrowArrayGuard`) that release exactly once, and -each batch outlives the stream that produced it: +Batched: `xl::to_arrow_stream` is `xl::to_arrow` delivered a batch at a time. `next()` returns +an `xl::ArrowTable` holding the batch and its own copy of the schema (`schema()` returns one whose +`array` is empty); each releases exactly once and outlives the stream that produced it: ```cpp -auto stream = xl::arrow_stream(workbook->sheet(0), 1, 10'000); +auto stream = xl::to_arrow_stream(workbook->sheet(0), {.batch_size = 10'000}); while (true) { auto batch = stream->next(); @@ -303,13 +297,13 @@ while (true) } ``` -Same `header_row`/`batch_size` meaning as `xl::typed_reader` above. +Same `header_row`/`batch_size` meaning as `xl::parse_batches` above. ## Writing Two layers, mirroring the two on the reading side. -`xl::write_sheet` uses the same `xl::ExcelMapper` specialization `xl::parse_sheet` reads +`xl::write_sheet` uses the same `xl::ExcelMapper` specialization `xl::parse` reads with, so a round trip needs one mapping, not two: ```cpp @@ -330,7 +324,15 @@ std::array columns{ xl::i64_column("id", ids), xl::f64_column("value", values)}; -auto written = xl::write_columns("out.xlsx", XL_FORMAT_XLSX, columns); +auto written = xl::write_columns("out.xlsx", columns); +``` + +Both functions also write into memory: pass a `std::vector&` instead of a path and it +is replaced with the file's bytes. There is no extension to infer the format from, so name it: + +```cpp +std::vector bytes; +auto written = xl::write_sheet(bytes, rows, {.format = XL_FORMAT_XLSX}); ``` One constructor per column type: `i64_column`, `f64_column`, `bool_column`, `date_column`, @@ -343,24 +345,65 @@ is long enough for the row count before calling: the ABI takes it without a leng `(rows + 7) / 8` bytes on trust, so a short one would be a buffer overrun. On the struct side, declare the field `std::optional` and `write_sheet` builds the bitmap for you. -`xl::WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. -`XL_FORMAT_AUTO` is rejected — a file being created has no signature bytes to sniff — so -`xl::format_from_path` returning `XL_FORMAT_AUTO` for an unrecognized extension surfaces as a failed -write rather than a silently chosen format. +`xl::WriteOptions` sets the format, the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB +toggles. Its `format` defaults to `XL_FORMAT_AUTO`, which takes the format from the path's +extension (.xlsx, .xlsb, .xls or .csv, in any case); an unrecognized extension, or AUTO with a vector +destination, fails the write rather than silently choosing a format. `write_sheet` walks the range once and appends each field to its own column buffer, with the per-field dispatch resolved at compile time. That transpose is the only copy it makes, and it is what the ABI's columnar shape costs a row-shaped caller; `write_columns` pays nothing. +### Writing cell by cell + +`xl::WriterHandle` writes a workbook as calls arrive, sheet by sheet, without holding it in memory +first. Call `finish()` at the end: destruction finishes too, but cannot report a failure. Opened over +a `std::vector&`, the handle fills the vector when it finishes. + +```cpp +auto writer = xl::WriterHandle::open("out.xlsx").value(); +writer.start_sheet("Totals"); +writer.start_row(); +writer.write(std::string_view("widgets")); +writer.write(int64_t{7}); +writer.write(std::optional{}); // a blank cell +writer.end_row(); +writer.end_sheet(); +if (auto done = writer.finish(); !done) { /* done.error().message */ } +``` + +Every call returns `std::expected`; one out of order fails instead of corrupting the +file. + +## Aggregating a CSV in parallel + +`xl::aggregate_csv` folds a CSV file or buffer across worker threads. Each partition starts from +a copy of the seed, `accumulate` sees each record, and `combine` merges partitions: + +```cpp +struct RowCount +{ + int64_t rows = 0; + int32_t accumulate(xl::RowView) noexcept { ++rows; return XL_OK; } + int32_t combine(RowCount &next) noexcept { rows += next.rows; return XL_OK; } +}; + +auto counted = xl::aggregate_csv("big.csv", RowCount{}, {.csv_delimiter = ';'}, /*header_row*/ 1); +``` + +The dialect (`csv_delimiter`, `csv_quote`, `csv_detect_bom`) and `max_cell_bytes` come from +`xl::OpenOptions`; the header row and the degree of parallelism (0 = every processor) follow it. + ## Bounds and ABI `TableView::operator[]` is unchecked, like `std::vector`'s. Use `TableView::at(row)`, which returns `std::optional` and is `nullopt` outside `[0, size())`. -`xl::Workbook::open`/`open_memory` first check the loaded library's `xl_abi_version()` against the -`XL_ABI_VERSION` this header was compiled against, and fail with an explanatory `xl::Error` rather -than reading native memory through a layout that may have changed. `xl::abi_version()` exposes the -loaded revision directly. +Every entry point that starts native work (`xl::Workbook::open`, `write_columns`, `write_sheet`, +`WriterHandle::open`, `encrypt_package` and `aggregate_csv`) first checks the loaded library's +`xl_abi_version()` against the `XL_ABI_VERSION` this header was compiled against, and fails with an +explanatory `xl::Error` rather than reading native memory through a layout that may have changed. +`xl::abi_version()` exposes the loaded revision directly. ## Benchmarks @@ -370,16 +413,16 @@ on Windows 10 (22H2), 16 logical CPUs @ 3.39 GHz, MSVC 19.51 (Release), Google B | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| -| `open` (`open_memory`) | 24.0 µs | 1.17 ms | -| `parse_sheet` (4 or 6 bound columns) | 47.8 µs | 24.7 ms | -| `typed_reader`, batches of 1,000 / 10,000 rows | - | 23.8 ms / 23.8 ms | +| `open` (from memory) | 24.0 µs | 1.17 ms | +| `parse` (4 or 6 bound columns) | 47.8 µs | 24.7 ms | +| `parse_batches`, batches of 1,000 / 10,000 rows | - | 23.8 ms / 23.8 ms | | `infer_schema` (sample 100 / 1,000 rows) | 75.6 µs | 579.1 µs | -`open` here is `open_memory`, which copies the caller's buffer before returning (the caller may free +`open` here is the byte-span overload, which copies the caller's buffer before returning (the caller may free it afterwards), so it scales with file size: 1.17 ms is mostly copying the 3.7 MB fixture. Opening by path reads only the package directory and stays nearly flat — see the Rust suite below. -`parse_sheet` scales linearly with rows × columns, at roughly 380 ns/row here. Draining the same -sheet through the batched typed reader costs the same as one whole-sheet `parse_sheet` while holding +`parse` scales linearly with rows × columns, at roughly 380 ns/row here. Draining the same +sheet through the batched typed reader costs the same as one whole-sheet `parse` while holding only one batch in memory at a time. `infer_schema` scales with its sample size, not the file's total row count. @@ -393,7 +436,7 @@ sides decode every cell into an owned value (`std::string` for text, not the zer | Library | Mean | |---|---:| -| ExcelReader (`parse_sheet`) | 62.9 ms | +| ExcelReader (`parse`) | 62.9 ms | | DuckDB (`read_xlsx`, summing every column in SQL) | 410.8 ms | | xlsxio (`xlsxioread_sheet_next_cell_*`) | 512.9 ms | | xlnt (`worksheet::rows()` + `cell::to_string()`/`value()`) | 2,394.1 ms | @@ -426,7 +469,7 @@ because the DuckDB comparison above only links under MSVC. Two workloads: -- **Typed:** `parse_sheet` against zsv plus the same conversions done by hand (`std::string` +- **Typed:** `parse` against zsv plus the same conversions done by hand (`std::string` for text, ISO dates, `std::from_chars` for the integers and doubles), one owned `FullRow` per row. All three produce the same checksum. - **Cells:** every cell's bytes counted. ExcelReader goes through `RowCursor` (`xl_rows_next_view`), zsv diff --git a/cpp/benchmarks/benchmark_compare.cpp b/cpp/benchmarks/benchmark_compare.cpp index 39fdb096..fc640333 100644 --- a/cpp/benchmarks/benchmark_compare.cpp +++ b/cpp/benchmarks/benchmark_compare.cpp @@ -103,13 +103,13 @@ static void BM_ExcelReader_Xlsx_Full(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); return; } - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); if (!table.has_value()) { state.SkipWithError(table.error().message); @@ -135,13 +135,13 @@ static void BM_ExcelReader_Xlsx_ParseOnly(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); return; } - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); if (!table.has_value()) { state.SkipWithError(table.error().message); diff --git a/cpp/benchmarks/benchmark_csv_compare.cpp b/cpp/benchmarks/benchmark_csv_compare.cpp index 211faf13..8890e48c 100644 --- a/cpp/benchmarks/benchmark_csv_compare.cpp +++ b/cpp/benchmarks/benchmark_csv_compare.cpp @@ -202,13 +202,13 @@ static void BM_ExcelReader_Csv_Full(benchmark::State &state) const std::vector buffer = read_fixture(); for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer, XL_FORMAT_CSV); + auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); return; } - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); if (!table.has_value()) { state.SkipWithError(table.error().message); @@ -253,7 +253,7 @@ static void BM_ExcelReader_Csv_Cells(benchmark::State &state) const std::vector buffer = read_fixture(); for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer, XL_FORMAT_CSV); + auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); diff --git a/cpp/benchmarks/benchmark_workbook.cpp b/cpp/benchmarks/benchmark_workbook.cpp index a80e4e5c..b0dcad82 100644 --- a/cpp/benchmarks/benchmark_workbook.cpp +++ b/cpp/benchmarks/benchmark_workbook.cpp @@ -95,7 +95,7 @@ static void BM_Open(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); benchmark::DoNotOptimize(workbook); if (!workbook.has_value()) { @@ -117,7 +117,7 @@ static void BM_ParseSheet(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -125,7 +125,7 @@ static void BM_ParseSheet(benchmark::State &state) } state.ResumeTiming(); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); benchmark::DoNotOptimize(table); if (!table.has_value()) { @@ -147,7 +147,7 @@ static void BM_InferSchema(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -177,7 +177,7 @@ static void BM_Open_Large(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); benchmark::DoNotOptimize(workbook); if (!workbook.has_value()) { @@ -199,7 +199,7 @@ static void BM_ParseSheet_Large(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -207,7 +207,7 @@ static void BM_ParseSheet_Large(benchmark::State &state) } state.ResumeTiming(); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); benchmark::DoNotOptimize(table); if (!table.has_value()) { @@ -229,7 +229,7 @@ static void BM_TypedReaderBatched_Large(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -237,7 +237,7 @@ static void BM_TypedReaderBatched_Large(benchmark::State &state) } state.ResumeTiming(); - auto reader = xl::typed_reader(workbook->sheet(0), 1, state.range(0)); + auto reader = xl::parse_batches(workbook->sheet(0), {.batch_size = state.range(0)}); if (!reader.has_value()) { state.SkipWithError(reader.error().message.c_str()); @@ -256,7 +256,7 @@ static void BM_TypedReaderBatched_Large(benchmark::State &state) benchmark::DoNotOptimize(rows); } } -BENCHMARK(BM_TypedReaderBatched_Large)->Arg(1000)->Arg(10000)->Arg(0); +BENCHMARK(BM_TypedReaderBatched_Large)->Arg(1000)->Arg(10000); static void BM_InferSchema_Large(benchmark::State &state) { @@ -269,7 +269,7 @@ static void BM_InferSchema_Large(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); diff --git a/cpp/benchmarks/benchmark_write.cpp b/cpp/benchmarks/benchmark_write.cpp index 40aea04c..c0ebf59d 100644 --- a/cpp/benchmarks/benchmark_write.cpp +++ b/cpp/benchmarks/benchmark_write.cpp @@ -45,13 +45,13 @@ static const std::vector &fixture_rows() std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_LARGE_FIXTURE_PATH); std::abort(); } - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); if (!table.has_value() || table->size() == 0) { std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_LARGE_FIXTURE_PATH); std::abort(); } - return table->to_vector(); + return std::vector(table->begin(), table->end()); }(); return rows; } @@ -89,7 +89,7 @@ static void BM_WriteSheet(benchmark::State &state) const std::filesystem::path path = bench_path("sheet.xlsx"); for (auto _ : state) { - auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + auto result = xl::write_sheet(path.string(), rows); benchmark::DoNotOptimize(result); if (!result.has_value()) { @@ -158,7 +158,7 @@ static void BM_WriteColumns(benchmark::State &state) const std::filesystem::path path = bench_path("columns.xlsx"); for (auto _ : state) { - auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + auto result = xl::write_columns(path.string(), columns); benchmark::DoNotOptimize(result); if (!result.has_value()) { diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp index 0e9eb191..9508a8d7 100644 --- a/cpp/benchmarks/benchmark_write_compare.cpp +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -77,19 +77,19 @@ namespace { static const std::vector rows = [] { - auto workbook = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH, XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH, {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_XLSX_FIXTURE_PATH); std::abort(); } - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); if (!table.has_value() || table->size() == 0) { std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_XLSX_FIXTURE_PATH); std::abort(); } - return table->to_vector(); + return std::vector(table->begin(), table->end()); }(); return rows; } @@ -132,7 +132,7 @@ static void BM_ExcelReader_WriteSheet(benchmark::State &state) const std::filesystem::path path = bench_path("sheet.xlsx"); for (auto _ : state) { - auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + auto result = xl::write_sheet(path.string(), rows); if (!result.has_value()) { state.SkipWithError(result.error().message.c_str()); @@ -216,7 +216,7 @@ static void BM_ExcelReader_WriteColumns(benchmark::State &state) const std::filesystem::path path = bench_path("columns.xlsx"); for (auto _ : state) { - auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + auto result = xl::write_columns(path.string(), columns); if (!result.has_value()) { state.SkipWithError(result.error().message.c_str()); diff --git a/cpp/tests/aggregate.cpp b/cpp/tests/aggregate.cpp index bc0841a7..b4c72147 100644 --- a/cpp/tests/aggregate.cpp +++ b/cpp/tests/aggregate.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -64,15 +65,19 @@ namespace static_assert(xl::CsvAccumulator, "RowCounter must satisfy xl::CsvAccumulator"); - template - concept SeedAccepted = requires(std::span data, Seed &seed) { - xl::aggregate_csv_memory(data, seed); + struct MoveOnly + { + std::unique_ptr state; + int32_t accumulate(xl::RowView) noexcept { return XL_OK; } + int32_t combine(MoveOnly &) noexcept { return XL_OK; } }; - static_assert(SeedAccepted, - "a const-callable seed must be accepted"); - static_assert(!SeedAccepted, - "a mutable seed must fail the constraint cleanly, not hard-error inside a shim"); + template + concept Aggregatable = requires(std::span data) { xl::aggregate_csv(data); }; + + static_assert(Aggregatable, "a copyable accumulator must be accepted"); + static_assert(!Aggregatable, + "every partition copies the seed, so a move-only accumulator must fail the constraint cleanly"); constexpr int64_t kLargeRowCount = 800'000; @@ -91,13 +96,9 @@ namespace static int test_aggregate_csv_memory() { - xl::CsvParallelOptions options{}; - options.header_row = 1; - options.degree_of_parallelism = 1; - std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data, &options); - CHECK(result.has_value(), "aggregate_csv_memory must succeed on a well-formed in-memory CSV"); + auto result = xl::aggregate_csv(data, RowCounter{}, {}, 1, 1); + CHECK(result.has_value(), "aggregate_csv must succeed on a well-formed in-memory CSV"); CHECK(result->rows == 3, "the fixture has 3 data rows after the header"); return 0; } @@ -111,14 +112,10 @@ static int test_aggregate_csv_file() out.write(kCsv.data(), static_cast(kCsv.size())); } - xl::CsvParallelOptions options{}; - options.header_row = 1; - options.degree_of_parallelism = 1; - - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 1, 1); std::filesystem::remove(path); - CHECK(result.has_value(), "aggregate_csv_file must succeed on a well-formed CSV file"); + CHECK(result.has_value(), "aggregate_csv must succeed on a well-formed CSV file"); CHECK(result->rows == 3, "the fixture has 3 data rows after the header"); return 0; } @@ -126,8 +123,8 @@ static int test_aggregate_csv_file() static int test_aggregate_default_options() { std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data); - CHECK(result.has_value(), "aggregate_csv_memory must succeed with default (null) options"); + auto result = xl::aggregate_csv(data); + CHECK(result.has_value(), "aggregate_csv must succeed with default options"); CHECK(result->rows == 4, "with no header configured, all 4 rows are data rows"); return 0; } @@ -138,13 +135,10 @@ static int test_aggregate_csv_file_partitioned() std::filesystem::temp_directory_path() / "excelreader-cpp-aggregate-large.csv"; write_repeated_rows(path, kLargeRowCount); - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 8; - - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 0, 8); std::filesystem::remove(path); - CHECK(result.has_value(), "aggregate_csv_file must succeed on a large multi-partition CSV file"); + CHECK(result.has_value(), "aggregate_csv must succeed on a large multi-partition CSV file"); CHECK(result->rows == kLargeRowCount, "every one of the generated rows must be counted exactly once"); CHECK(result->combines >= 1, @@ -173,9 +167,7 @@ static int test_aggregate_quoted_records_straddling_chunks() out.write(content.data(), static_cast(content.size())); } - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 8; - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 0, 8); std::filesystem::remove(path); CHECK(result.has_value(), "a file of quoted multi-line records must aggregate"); @@ -194,14 +186,14 @@ static int test_aggregate_reports_a_throwing_accumulate() }; std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data); + auto result = xl::aggregate_csv(data); CHECK(!result.has_value(), "a throwing accumulate must produce an error, not a value"); CHECK(std::string_view(result.error().message).find("threw") != std::string_view::npos, "the error must say a callback threw"); return 0; } -static int test_aggregate_with_a_seed_callable() +static int test_aggregate_with_a_seed() { struct Summer { @@ -226,14 +218,12 @@ static int test_aggregate_with_a_seed_callable() const int64_t step = 10; std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - xl::CsvParallelOptions options{}; - options.header_row = 1; - auto result = xl::aggregate_csv_memory(data, [step] { return Summer{0, step}; }, &options); - CHECK(result.has_value(), "aggregate_csv_memory with a seed callable must succeed"); + auto result = xl::aggregate_csv(data, Summer{0, step}, {}, 1); + CHECK(result.has_value(), "aggregate_csv with a seed must succeed"); CHECK(result->total == 30, "3 data rows at step 10 must total 30"); - auto no_options = xl::aggregate_csv_memory(data, [step] { return Summer{0, step}; }); - CHECK(no_options.has_value(), "a seed callable with default (null) options must succeed"); + auto no_options = xl::aggregate_csv(data, Summer{0, step}); + CHECK(no_options.has_value(), "a seed with default options must succeed"); CHECK(no_options->total == 40, "with no header configured, all 4 rows are data rows"); const std::filesystem::path path = @@ -242,9 +232,9 @@ static int test_aggregate_with_a_seed_callable() std::ofstream out(path, std::ios::binary); out.write(kCsv.data(), static_cast(kCsv.size())); } - auto from_file = xl::aggregate_csv_file(path.string(), [step] { return Summer{0, step}; }); + auto from_file = xl::aggregate_csv(path.string(), Summer{0, step}); std::filesystem::remove(path); - CHECK(from_file.has_value(), "aggregate_csv_file with a seed callable must succeed"); + CHECK(from_file.has_value(), "aggregate_csv of a file with a seed must succeed"); CHECK(from_file->total == 40, "with no header configured, all 4 rows are data rows"); return 0; } @@ -257,11 +247,8 @@ static int test_aggregate_reports_a_callback_status_of_int32_max() int32_t combine(ReturnsIntMax &) noexcept { return XL_OK; } }; - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 1; - std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data, &options); + auto result = xl::aggregate_csv(data, {}, {}, 0, 1); CHECK(!result.has_value(), "a nonzero callback status must abort the run"); CHECK(result.error().code == (std::numeric_limits::max)(), "the caller's own status must come back verbatim"); @@ -283,10 +270,7 @@ static int test_aggregate_destroys_every_state() g_constructed.store(0); g_destructed.store(0); { - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 8; - - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 0, 8); CHECK(result.has_value(), "the run must succeed"); CHECK(result->combines >= 1, "the fixture must really be split, or free_state never runs and the balance is " @@ -300,6 +284,35 @@ static int test_aggregate_destroys_every_state() return 0; } +static int test_aggregate_takes_the_dialect_from_open_options() +{ + struct CellCounter + { + int64_t cells = 0; + + int32_t accumulate(xl::RowView row) noexcept + { + cells += static_cast(row.size()); + return XL_OK; + } + + int32_t combine(CellCounter &next) noexcept + { + cells += next.cells; + return XL_OK; + } + }; + + constexpr std::string_view semicolons = "a;b\n1;2\n3;4\n"; + std::span data(reinterpret_cast(semicolons.data()), semicolons.size()); + auto split = xl::aggregate_csv(data, CellCounter{}, {.csv_delimiter = ';'}, 1, 1); + CHECK(split.has_value() && split->cells == 4, "csv_delimiter must split each data row into two cells"); + + auto unsplit = xl::aggregate_csv(data, CellCounter{}, {}, 1, 1); + CHECK(unsplit.has_value() && unsplit->cells == 2, "the default delimiter leaves each row one cell"); + return 0; +} + int main() { if (int failed = test_aggregate_csv_memory()) @@ -330,7 +343,11 @@ int main() { return failed; } - if (int failed = test_aggregate_with_a_seed_callable()) + if (int failed = test_aggregate_with_a_seed()) + { + return failed; + } + if (int failed = test_aggregate_takes_the_dialect_from_open_options()) { return failed; } diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp index f42ca3c8..73639841 100644 --- a/cpp/tests/arrow.cpp +++ b/cpp/tests/arrow.cpp @@ -34,12 +34,12 @@ struct xl::ExcelMapper int main() { - xl::OpenOptions options{.prefetch_decompression = 1}; - auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + const xl::OpenOptions options{.format = XL_FORMAT_XLSB, .prefetch_decompression = XL_OPT_TRUE}; + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(workbook.has_value(), "xl::Workbook::open must succeed on the RealExcel.xlsb fixture"); - auto table = xl::parse_arrow(workbook->sheet(0)); - CHECK(table.has_value(), "xl::parse_arrow must succeed"); + auto table = xl::to_arrow(workbook->sheet(0)); + CHECK(table.has_value(), "xl::to_arrow must succeed"); CHECK(std::strcmp(table->schema.format, "+s") == 0, "top level must be a struct array"); CHECK(table->schema.n_children == 2, "must have two child columns"); @@ -55,19 +55,20 @@ int main() CHECK(table->array.release == nullptr, "moved-from table must be released/inert"); } - auto failed = xl::parse_arrow(workbook->sheet(0), 1'000'000); - CHECK(!failed.has_value(), "xl::parse_arrow must fail for an out-of-range header_row"); + auto failed = xl::to_arrow(workbook->sheet(0), {.header_row = 1'000'000}); + CHECK(!failed.has_value(), "xl::to_arrow must fail for an out-of-range header_row"); { - auto stream_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto stream_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(stream_workbook.has_value(), "opening a workbook for the Arrow stream must succeed"); - auto stream = xl::arrow_stream(stream_workbook->sheet(0), 1, 8); - CHECK(stream.has_value(), "xl::arrow_stream must succeed"); + auto stream = xl::to_arrow_stream(stream_workbook->sheet(0), {.batch_size = 8}); + CHECK(stream.has_value(), "xl::to_arrow_stream must succeed"); auto schema = stream->schema(); CHECK(schema.has_value(), "the stream must report a schema"); CHECK(schema->schema.release != nullptr, "the schema must be a live, owned struct"); + CHECK(schema->array.release == nullptr, "schema() must leave the table's array empty"); CHECK(std::strcmp(schema->schema.format, "+s") == 0, "the stream schema must be a struct array"); CHECK(schema->schema.n_children == 2, "the stream schema must have two child columns"); @@ -83,22 +84,24 @@ int main() } ++batches; CHECK((*batch)->array.length <= 8, "no batch may exceed the requested batch size"); + CHECK((*batch)->schema.release != nullptr && (*batch)->schema.n_children == 2, + "every batch must carry its own schema"); rows += (*batch)->array.length; } CHECK(rows == 100, "the stream must deliver all 100 data rows"); CHECK(batches == 13, "100 rows at batch size 8 is 13 batches"); - auto second = xl::arrow_stream(stream_workbook->sheet(0), 1, 8); + auto second = xl::to_arrow_stream(stream_workbook->sheet(0), {.batch_size = 8}); CHECK(second.has_value(), "a second stream on one workbook opens"); } { - auto abandoned_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto abandoned_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(abandoned_workbook.has_value(), "opening a workbook for the abandonment test must succeed"); - xl::ArrowArrayGuard kept; + xl::ArrowTable kept; { - auto stream = xl::arrow_stream(abandoned_workbook->sheet(0), 1, 4); + auto stream = xl::to_arrow_stream(abandoned_workbook->sheet(0), {.batch_size = 4}); CHECK(stream.has_value(), "the abandoned stream must open"); auto batch = stream->next(); CHECK(batch.has_value() && batch->has_value(), "one batch must read before abandonment"); @@ -107,42 +110,30 @@ int main() CHECK(kept.array.length == 4, "a batch must outlive the stream that produced it"); kept.release(); kept.release(); - CHECK(kept.array.release == nullptr, "releasing must null the release callback"); + CHECK(kept.array.release == nullptr && kept.schema.release == nullptr, "releasing must null both release callbacks"); } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); - CHECK(wb.has_value(), "opening a workbook for the batch_size=0 stream must succeed"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, 0); - CHECK(stream.has_value(), "xl::arrow_stream with batch_size=0 must succeed"); - auto batch = stream->next(); - CHECK(batch.has_value() && batch->has_value(), "batch_size=0 must yield one batch"); - CHECK((*batch)->array.length == 100, "the one batch must hold all 100 rows"); - auto end = stream->next(); - CHECK(end.has_value() && !end->has_value(), "the stream ends after the single batch"); - } - - { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); - CHECK(wb.has_value(), "opening a workbook for the negative-batch-size stream must succeed"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, -1); - CHECK(!stream.has_value(), "a negative batch size must be rejected"); - if (!stream.has_value()) + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); + CHECK(wb.has_value(), "opening a workbook for the non-positive-batch-size stream must succeed"); + for (int64_t batch_size : {int64_t{0}, int64_t{-1}}) { + auto stream = xl::to_arrow_stream(wb->sheet(0), {.batch_size = batch_size}); + CHECK(!stream.has_value(), "a batch size below 1 must be rejected"); CHECK(stream.error().code == XL_INVALID_ARGUMENT, "the rejection is XL_INVALID_ARGUMENT"); } } { - auto whole_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto whole_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(whole_wb.has_value(), "opening a workbook for the whole-sheet comparison must succeed"); - auto whole = xl::parse_arrow(whole_wb->sheet(0)); - CHECK(whole.has_value(), "the whole-sheet parse_arrow must succeed for comparison"); + auto whole = xl::to_arrow(whole_wb->sheet(0)); + CHECK(whole.has_value(), "the whole-sheet to_arrow must succeed for comparison"); - auto stream_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto stream_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(stream_wb.has_value(), "opening a workbook for the streamed comparison must succeed"); - auto stream = xl::arrow_stream(stream_wb->sheet(0), 1, 4); - CHECK(stream.has_value(), "xl::arrow_stream at batch_size=4 must succeed"); + auto stream = xl::to_arrow_stream(stream_wb->sheet(0), {.batch_size = 4}); + CHECK(stream.has_value(), "xl::to_arrow_stream at batch_size=4 must succeed"); auto first = stream->next(); CHECK(first.has_value() && first->has_value(), "the first streamed batch must read"); @@ -177,14 +168,14 @@ int main() } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(wb.has_value(), "opening a workbook for the foreign-read test must succeed"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, 4); + auto stream = xl::to_arrow_stream(wb->sheet(0), {.batch_size = 4}); CHECK(stream.has_value(), "the stream opens before the foreign read"); auto first = stream->next(); CHECK(first.has_value() && first->has_value(), "the first batch reads before the foreign read"); - auto stolen = xl::parse_sheet(wb->sheet(0)); + auto stolen = xl::parse(wb->sheet(0)); CHECK(stolen.has_value(), "the foreign whole-sheet read succeeds"); auto after = stream->next(); @@ -192,19 +183,19 @@ int main() } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(wb.has_value(), "opening a workbook for the reader-then-stream test must succeed"); - auto reader = xl::typed_reader(wb->sheet(0), 1, 8); + auto reader = xl::parse_batches(wb->sheet(0), {.batch_size = 8}); CHECK(reader.has_value(), "the typed reader opens first"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, 8); + auto stream = xl::to_arrow_stream(wb->sheet(0), {.batch_size = 8}); CHECK(stream.has_value(), "an Arrow stream opens while a typed reader is live"); } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(wb.has_value(), "opening a workbook for the stream-then-reader test must succeed"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, 8); + auto stream = xl::to_arrow_stream(wb->sheet(0), {.batch_size = 8}); CHECK(stream.has_value(), "the Arrow stream opens first"); - auto reader = xl::typed_reader(wb->sheet(0), 1, 8); + auto reader = xl::parse_batches(wb->sheet(0), {.batch_size = 8}); CHECK(reader.has_value(), "a typed reader opens while an Arrow stream is live"); } diff --git a/cpp/tests/batch.cpp b/cpp/tests/batch.cpp index d8cc1b2c..67e2a6f7 100644 --- a/cpp/tests/batch.cpp +++ b/cpp/tests/batch.cpp @@ -47,8 +47,8 @@ namespace { return {}; } - auto table = xl::parse_sheet(workbook->sheet(0)); - check(table.has_value(), "parse_sheet succeeds"); + auto table = xl::parse(workbook->sheet(0)); + check(table.has_value(), "parse succeeds"); if (!table.has_value()) { return {}; @@ -65,8 +65,7 @@ namespace void test_batches_equal_whole_sheet(const std::vector &expected) { const int64_t rows = static_cast(expected.size()); - for (int64_t batch_size : {int64_t{1}, int64_t{7}, int64_t{8}, int64_t{9}, int64_t{1000}, - int64_t{0}, rows + 10}) + for (int64_t batch_size : {int64_t{1}, int64_t{7}, int64_t{8}, int64_t{9}, int64_t{1000}, rows + 10}) { auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH); check(workbook.has_value(), "open the fixture for a batched read"); @@ -74,8 +73,8 @@ namespace { return; } - auto reader = xl::typed_reader(workbook->sheet(0), 1, batch_size); - check(reader.has_value(), "typed_reader opens"); + auto reader = xl::parse_batches(workbook->sheet(0), {.batch_size = batch_size}); + check(reader.has_value(), "parse_batches opens"); if (!reader.has_value()) { return; @@ -92,10 +91,7 @@ namespace break; } ++batches; - if (batch_size > 0) - { - check((*batch)->size() <= batch_size, "no batch exceeds the requested size"); - } + check((*batch)->size() <= batch_size, "no batch exceeds the requested size"); for (Row row : **batch) { actual.emplace_back(std::string(row.Coluna1), row.Coluna3); @@ -104,7 +100,7 @@ namespace check(actual == expected, "batched rows match the whole-sheet read"); - const int64_t want = batch_size == 0 ? 1 : (rows + batch_size - 1) / batch_size; + const int64_t want = (rows + batch_size - 1) / batch_size; check(batches == want, "batch count matches ceil(rows / batch_size)"); } } @@ -117,8 +113,8 @@ namespace { return; } - auto reader = xl::typed_reader(workbook->sheet(0), 1, 8); - check(reader.has_value(), "typed_reader opens for the range-for read"); + auto reader = xl::parse_batches(workbook->sheet(0), {.batch_size = 8}); + check(reader.has_value(), "parse_batches opens for the range-for read"); if (!reader.has_value()) { return; @@ -145,25 +141,28 @@ namespace { return; } - auto first = xl::typed_reader(workbook->sheet(0), 1, 8); + auto first = xl::parse_batches(workbook->sheet(0), {.batch_size = 8}); check(first.has_value(), "the first reader opens"); - auto second = xl::typed_reader(workbook->sheet(0), 1, 8); + auto second = xl::parse_batches(workbook->sheet(0), {.batch_size = 8}); check(second.has_value(), "a second reader on the same workbook opens"); } - void test_negative_batch_size_is_rejected() + void test_non_positive_batch_size_is_rejected() { auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH); - check(workbook.has_value(), "open the fixture for the negative-batch test"); + check(workbook.has_value(), "open the fixture for the non-positive-batch test"); if (!workbook.has_value()) { return; } - auto reader = xl::typed_reader(workbook->sheet(0), 1, -1); - check(!reader.has_value(), "a negative batch size is rejected"); - if (!reader.has_value()) + for (int64_t batch_size : {int64_t{0}, int64_t{-1}}) { - check(reader.error().code == XL_INVALID_ARGUMENT, "the rejection is XL_INVALID_ARGUMENT"); + auto reader = xl::parse_batches(workbook->sheet(0), {.batch_size = batch_size}); + check(!reader.has_value(), "a batch size below 1 is rejected"); + if (!reader.has_value()) + { + check(reader.error().code == XL_INVALID_ARGUMENT, "the rejection is XL_INVALID_ARGUMENT"); + } } } @@ -175,7 +174,7 @@ namespace { return; } - auto reader = xl::typed_reader(workbook->sheet(0), 1, 4); + auto reader = xl::parse_batches(workbook->sheet(0), {.batch_size = 4}); check(reader.has_value(), "the reader opens before the foreign read"); if (!reader.has_value()) { @@ -184,7 +183,7 @@ namespace auto first = reader->next(); check(first.has_value() && first->has_value(), "the first batch reads before the foreign read"); - auto stolen = xl::parse_sheet(workbook->sheet(0)); + auto stolen = xl::parse(workbook->sheet(0)); check(stolen.has_value(), "the foreign whole-sheet read succeeds"); auto after = reader->next(); @@ -199,7 +198,7 @@ int main() test_batches_equal_whole_sheet(expected); test_range_for_matches_next(expected); test_second_reader_is_allowed(); - test_negative_batch_size_is_rejected(); + test_non_positive_batch_size_is_rejected(); test_foreign_read_leaves_the_reader_alone(); if (failures != 0) diff --git a/cpp/tests/package/main.cpp b/cpp/tests/package/main.cpp index 2aaed760..3e6f1c7a 100644 --- a/cpp/tests/package/main.cpp +++ b/cpp/tests/package/main.cpp @@ -4,7 +4,7 @@ int main() { - auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSX, nullptr); + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { std::fprintf(stderr, "FAIL: the installed package could not open %s\n", EXCELREADER_FIXTURE_PATH); diff --git a/cpp/tests/rows.cpp b/cpp/tests/rows.cpp index 587cfe85..655c457b 100644 --- a/cpp/tests/rows.cpp +++ b/cpp/tests/rows.cpp @@ -86,7 +86,7 @@ int main() out << "a\n" << std::string(200000, 'x') << "\n"; } - auto workbook = xl::Workbook::open(path.string(), XL_FORMAT_CSV); + auto workbook = xl::Workbook::open(path.string(), {.format = XL_FORMAT_CSV}); check(workbook.has_value(), "open the oversized CSV"); if (workbook.has_value()) { @@ -107,17 +107,25 @@ int main() check((*big)[0].type == xl::CellType::String, "the oversized cell is a string"); } } + + auto limited = xl::Workbook::open(path.string(), {.format = XL_FORMAT_CSV, .max_cell_bytes = 1000}); + check(limited.has_value(), "open the oversized CSV under a cell limit"); + if (limited.has_value()) + { + auto all = limited->sheet(0).read_all(); + check(!all.has_value(), "max_cell_bytes bounds a CSV cell too"); + } std::error_code ignored; std::filesystem::remove(path, ignored); } { auto workbook = xl::Workbook::open(fixture()); - check(workbook.has_value(), "open for read_all_decoded"); + check(workbook.has_value(), "open for read_all"); if (workbook.has_value()) { - auto decoded = workbook->sheet(0).read_all_decoded(); - check(decoded.has_value(), "read_all_decoded succeeds"); + auto decoded = workbook->sheet(0).read_all(); + check(decoded.has_value(), "read_all succeeds"); if (decoded.has_value()) { check(decoded->size() == rows.size(), "same row count as the cursor"); @@ -148,7 +156,7 @@ int main() while (cursor.has_value() && cursor->next_row().has_value()) { } - auto decoded = workbook->sheet(0).read_all_decoded(); + auto decoded = workbook->sheet(0).read_all(); check(decoded.has_value(), "a read after another cursor drained is not an error"); check(decoded.has_value() && decoded->size() == rows.size(), "and it still reads every row, from the start"); } @@ -158,7 +166,7 @@ int main() auto workbook = xl::Workbook::open(fixture()); if (workbook.has_value()) { - auto decoded = workbook->sheet(0).read_all_decoded(); + auto decoded = workbook->sheet(0).read_all(); check(decoded.has_value(), "open a set to move"); if (decoded.has_value()) { diff --git a/cpp/tests/sheets.cpp b/cpp/tests/sheets.cpp index ea302817..9942daef 100644 --- a/cpp/tests/sheets.cpp +++ b/cpp/tests/sheets.cpp @@ -25,7 +25,8 @@ namespace std::vector build(int sheet_count, int row_count) { - auto writer = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); + std::vector bytes; + auto writer = xl::WriterHandle::open(bytes, {.format = XL_FORMAT_XLSX}); check(writer.has_value(), "open the in-memory writer"); if (!writer.has_value()) { @@ -44,9 +45,8 @@ namespace } check(writer->end_sheet().has_value(), "end a sheet"); } - auto bytes = writer->bytes(); - check(bytes.has_value(), "read the written bytes"); - return bytes.has_value() ? std::move(*bytes) : std::vector{}; + check(writer->finish().has_value(), "finish the in-memory writer"); + return bytes; } std::vector drain(xl::RowCursor &cursor) @@ -77,7 +77,7 @@ int main() const std::vector two = build(2, 3); { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open the two-sheet workbook"); if (workbook.has_value()) { @@ -109,22 +109,15 @@ int main() auto negative = workbook->sheet(-1).rows(); check(!negative.has_value() && negative.error().code == XL_INVALID_ARGUMENT, "a negative sheet index is an invalid argument"); - auto decoded = workbook->sheet(1).read_all_decoded(); - check(decoded.has_value() && decoded->size() == 3, "read_all_decoded reads the whole named sheet"); - auto partial = workbook->sheet(1).rows(); - check(partial.has_value() && partial->next_row().has_value(), "open a cursor and read one row"); - if (partial.has_value()) - { - auto rest = partial->read_all_decoded(); - check(rest.has_value() && rest->size() == 2, "the cursor's read_all_decoded returns the rows it has left"); - } + auto decoded = workbook->sheet(1).read_all(); + check(decoded.has_value() && decoded->size() == 3, "read_all reads the whole named sheet"); auto schema = workbook->sheet(1).infer_schema(0, 10); check(schema.has_value() && schema->size() == 1, "infer_schema samples the named sheet"); } } { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for two cursors"); if (workbook.has_value()) { @@ -142,7 +135,7 @@ int main() } { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for the moved cursor"); if (workbook.has_value()) { @@ -151,7 +144,7 @@ int main() if (cursor.has_value()) { xl::RowCursor moved = std::move(*cursor); - check(cursor->handle() == nullptr, "a moved-from cursor holds nothing"); + check(cursor->next_row().error().code == XL_INVALID_HANDLE, "a moved-from cursor holds nothing"); check(drain(moved).size() == 3, "the moved-to cursor reads every row"); } check(read_sheet(workbook->sheet(0)).size() == 3, "the workbook still reads after the cursors are gone"); @@ -159,7 +152,7 @@ int main() } { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for the move-assigned cursor"); if (workbook.has_value()) { @@ -170,7 +163,7 @@ int main() { check(a->next_row().has_value(), "the overwritten cursor advances"); *a = std::move(*b); - check(b->handle() == nullptr, "a move-assigned-from cursor holds nothing"); + check(b->next_row().error().code == XL_INVALID_HANDLE, "a move-assigned-from cursor holds nothing"); const auto values = drain(*a); check(values.size() == 3 && values.front() == "s1-r0", "the move-assigned cursor reads the other sheet from the top"); } @@ -179,11 +172,11 @@ int main() { const std::vector empty_book = build(1, 0); - auto workbook = xl::Workbook::open_memory(empty_book); + auto workbook = xl::Workbook::open(empty_book); check(workbook.has_value(), "open the empty workbook"); if (workbook.has_value()) { - auto decoded = workbook->sheet(0).read_all_decoded(); + auto decoded = workbook->sheet(0).read_all(); check(decoded.has_value() && decoded->empty(), "an empty sheet decodes to no rows"); } } @@ -192,7 +185,7 @@ int main() std::optional survivor; std::optional stale; { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for the outliving cursor"); if (workbook.has_value()) { @@ -220,7 +213,7 @@ int main() constexpr int sheet_count = 6; constexpr int row_count = 500; const std::vector many = build(sheet_count, row_count); - auto workbook = xl::Workbook::open_memory(many); + auto workbook = xl::Workbook::open(many); check(workbook.has_value(), "open the many-sheet workbook"); if (workbook.has_value()) { diff --git a/cpp/tests/smoke.cpp b/cpp/tests/smoke.cpp index 4dd245ee..e2e07cbb 100644 --- a/cpp/tests/smoke.cpp +++ b/cpp/tests/smoke.cpp @@ -53,8 +53,8 @@ struct xl::ExcelMapper static int test_parse(xl::Workbook &workbook) { - auto table = xl::parse_sheet(workbook.sheet(0)); - CHECK(table.has_value(), "xl::parse_sheet must succeed"); + auto table = xl::parse(workbook.sheet(0)); + CHECK(table.has_value(), "xl::parse must succeed"); CHECK(table->size() == 100, "RealExcel.xlsb has 100 data rows"); auto it = table->begin(); @@ -75,17 +75,14 @@ static int test_parse(xl::Workbook &workbook) static int test_sheets(xl::Workbook &workbook) { - auto count = workbook.sheet_count(); - CHECK(count.has_value(), "sheet_count must succeed"); - CHECK(*count >= 1, "the fixture has at least one sheet"); + auto sheets = workbook.sheets(); + CHECK(sheets.has_value(), "sheets must succeed"); + CHECK(!sheets->empty(), "the fixture has at least one sheet"); - auto names = workbook.sheet_names(); - CHECK(names.has_value(), "sheet_names must succeed"); - CHECK(names->size() == static_cast(*count), "sheet_names must return one name per sheet"); - - auto first = workbook.sheet_name_at(0); - CHECK(first.has_value(), "sheet_name_at(0) must succeed"); - CHECK(*first == names->front(), "sheet_name_at(0) must be the first of sheet_names"); + auto first = sheets->front().name(); + CHECK(first.has_value(), "the first sheet's name must be readable"); + auto same = workbook.sheet(0).name(); + CHECK(same.has_value() && *same == *first, "sheet(0) must be the first of sheets()"); CHECK(workbook.is_date1904().has_value(), "is_date1904 must succeed"); return 0; @@ -104,8 +101,8 @@ static int test_infer_schema(const xl::Workbook &workbook) static int test_parse_with_alias(xl::Workbook &workbook) { - auto table = xl::parse_sheet(workbook.sheet(0)); - CHECK(table.has_value(), "xl::parse_sheet must succeed by resolving the second candidate name"); + auto table = xl::parse(workbook.sheet(0)); + CHECK(table.has_value(), "xl::parse must succeed by resolving the second candidate name"); CHECK(table->size() == 100, "RealExcel.xlsb has 100 data rows"); AliasRow first = *table->begin(); return 0; @@ -119,19 +116,18 @@ static std::string encrypted_fixture_path(std::string_view filename) static int test_password_open() { xl::OpenOptions options{}; - options.password("hunter2"); - auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), XL_FORMAT_AUTO, &options); + options.password = "hunter2"; + auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), options); CHECK(workbook.has_value(), "opening an encrypted workbook with the correct password must succeed"); - auto count = workbook->sheet_count(); - CHECK(count.has_value(), "sheet_count must succeed on the decrypted workbook"); - CHECK(*count > 0, "the decrypted workbook must have at least one sheet"); + auto sheets = workbook->sheets(); + CHECK(sheets.has_value(), "sheets must succeed on the decrypted workbook"); + CHECK(!sheets->empty(), "the decrypted workbook must have at least one sheet"); return 0; } static int test_password_required() { - xl::OpenOptions options{}; - auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), XL_FORMAT_AUTO, &options); + auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx")); CHECK(!workbook.has_value(), "opening an encrypted workbook without a password must fail"); CHECK(workbook.error().code == XL_STATUS_PASSWORD_REQUIRED, "the failure must report XL_STATUS_PASSWORD_REQUIRED"); @@ -140,9 +136,7 @@ static int test_password_required() static int test_password_incorrect() { - xl::OpenOptions options{}; - options.password("wrong"); - auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), XL_FORMAT_AUTO, &options); + auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), {.password = "wrong"}); CHECK(!workbook.has_value(), "opening an encrypted workbook with the wrong password must fail"); CHECK(workbook.error().code == XL_STATUS_PASSWORD_INCORRECT, "the failure must report XL_STATUS_PASSWORD_INCORRECT"); @@ -153,14 +147,21 @@ int main() { CHECK(xl::abi_version() == XL_ABI_VERSION, "the linked native library must speak this header's ABI revision"); - xl::OpenOptions options{.prefetch_decompression = 1}; - auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + xl::OpenOptions options{.format = XL_FORMAT_XLSB, .prefetch_decompression = XL_OPT_TRUE}; + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(workbook.has_value(), "xl::Workbook::open must succeed on the RealExcel.xlsb fixture"); auto missing = xl::Workbook::open("does-not-exist.xlsx"); CHECK(!missing.has_value(), "opening a missing file must fail"); CHECK(!missing.error().message.empty(), "a failure must carry the native error detail"); + auto wrong_format = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, {.format = XL_FORMAT_XLSX}); + CHECK(!wrong_format.has_value(), "OpenOptions::format must reach the open: an XLSB is not an XLSX"); + + auto bad_limit = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, {.max_cell_bytes = -1}); + CHECK(!bad_limit.has_value() && bad_limit.error().code == XL_INVALID_ARGUMENT, + "an out-of-range limit must fail the open with XL_INVALID_ARGUMENT"); + if (int failed = test_sheets(*workbook)) { return failed; diff --git a/cpp/tests/sources.cpp b/cpp/tests/sources.cpp index a84afe35..76c265d9 100644 --- a/cpp/tests/sources.cpp +++ b/cpp/tests/sources.cpp @@ -127,29 +127,27 @@ namespace std::vector build_sheets(int sheet_count, int row_count) { - auto writer = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); - std::vector bytes; - if (!writer.has_value()) + std::vector written; { - return bytes; - } - for (int s = 0; s < sheet_count; ++s) - { - (void)writer->start_sheet("sheet" + std::to_string(s)); - for (int r = 0; r < row_count; ++r) + auto writer = xl::WriterHandle::open(written, {.format = XL_FORMAT_XLSX}); + if (!writer.has_value()) { - (void)writer->start_row(); - (void)writer->write("s" + std::to_string(s) + "-r" + std::to_string(r)); - (void)writer->end_row(); + return {}; + } + for (int s = 0; s < sheet_count; ++s) + { + (void)writer->start_sheet("sheet" + std::to_string(s)); + for (int r = 0; r < row_count; ++r) + { + (void)writer->start_row(); + (void)writer->write("s" + std::to_string(s) + "-r" + std::to_string(r)); + (void)writer->end_row(); + } + (void)writer->end_sheet(); } - (void)writer->end_sheet(); - } - auto written = writer->bytes(); - if (written.has_value()) - { - bytes.resize(written->size()); - std::memcpy(bytes.data(), written->data(), written->size()); } + std::vector bytes(written.size()); + std::memcpy(bytes.data(), written.data(), written.size()); return bytes; } @@ -171,7 +169,7 @@ namespace std::size_t row_count(const xl::Workbook &workbook) { - auto decoded = workbook.sheet(0).read_all_decoded(); + auto decoded = workbook.sheet(0).read_all(); return decoded.has_value() ? decoded->size() : 0; } } @@ -186,9 +184,9 @@ int main() Counters counters; { - auto workbook = xl::Workbook::open_source(std::make_unique(bytes, counters)); - check(workbook.has_value(), "open_source opens the fixture"); - check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_source reads every row"); + auto workbook = xl::Workbook::open(std::make_unique(bytes, counters)); + check(workbook.has_value(), "a Source opens the fixture"); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "a Source reads every row"); } check(counters.destroyed.load() == 1, "the source is destroyed exactly once"); @@ -197,29 +195,28 @@ int main() Counters uncached; { - xl::OpenOptions options; - options.source_block_size = -1; - auto workbook = xl::Workbook::open_source(std::make_unique(bytes, uncached), XL_FORMAT_XLSX, &options); - check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_source without the cache reads every row"); + auto workbook = xl::Workbook::open(std::make_unique(bytes, uncached), + {.format = XL_FORMAT_XLSX, .source_block_size = -1}); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "a Source without the cache reads every row"); } check(uncached.reads.load() > blocks, "without the cache every read reaches read_at"); { Counters failing; - auto workbook = xl::Workbook::open_source(std::make_unique(failing), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(std::make_unique(failing), {.format = XL_FORMAT_XLSX}); check(!workbook.has_value() && workbook.error().message.find("disk on fire") != std::string::npos, "the source's error message reaches the caller"); check(failing.destroyed.load() == 1, "a source whose read failed is destroyed exactly once"); } { Counters throwing; - auto workbook = xl::Workbook::open_source(std::make_unique(throwing), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(std::make_unique(throwing), {.format = XL_FORMAT_XLSX}); check(!workbook.has_value() && workbook.error().message.find("boom") != std::string::npos, "an exception from the source becomes an error"); check(throwing.destroyed.load() == 1, "a source that threw is destroyed exactly once"); } { - auto workbook = xl::Workbook::open_stream(std::make_unique(), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(std::make_unique(), {.format = XL_FORMAT_XLSX}); check(!workbook.has_value() && workbook.error().message.find("pipe broke") != std::string::npos, "the stream's error message reaches the caller"); } @@ -228,8 +225,8 @@ int main() const std::vector many = build_sheets(sheet_count, 300); check(!many.empty(), "build the multi-sheet workbook"); Counters parallel_counters; - auto reference_many = xl::Workbook::open_memory(std::span(reinterpret_cast(many.data()), many.size())); - auto workbook = xl::Workbook::open_source(std::make_unique(many, parallel_counters), XL_FORMAT_XLSX); + auto reference_many = xl::Workbook::open(std::span(reinterpret_cast(many.data()), many.size())); + auto workbook = xl::Workbook::open(std::make_unique(many, parallel_counters), {.format = XL_FORMAT_XLSX}); check(reference_many.has_value() && workbook.has_value(), "open the multi-sheet workbook both ways"); if (reference_many.has_value() && workbook.has_value()) { @@ -259,12 +256,12 @@ int main() const std::string csv = "name,qty\nwidget,7\n"; std::vector csv_bytes(csv.size()); std::memcpy(csv_bytes.data(), csv.data(), csv.size()); - auto workbook = xl::Workbook::open_stream(std::make_unique(csv_bytes), XL_FORMAT_CSV); - check(workbook.has_value() && row_count(*workbook) == 2, "open_stream reads a CSV"); + auto workbook = xl::Workbook::open(std::make_unique(csv_bytes), {.format = XL_FORMAT_CSV}); + check(workbook.has_value() && row_count(*workbook) == 2, "an InputStream reads a CSV"); } { - auto workbook = xl::Workbook::open_stream(std::make_unique(bytes)); - check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_stream reads an XLSX"); + auto workbook = xl::Workbook::open(std::make_unique(bytes)); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "an InputStream reads an XLSX"); } if (failures != 0) diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index b4809485..9381e135 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #define CHECK(cond, msg) \ do \ @@ -69,12 +70,12 @@ static int test_write_columns_round_trip() xl::timestamp_column("instante", instantes)}; const std::filesystem::path path = temp_path("columns.xlsx"); - auto written = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + auto written = xl::write_columns(path.string(), columns); CHECK(written.has_value(), "write_columns must succeed"); { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the written file must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 2, "two rows were written"); @@ -114,18 +115,16 @@ static int test_encrypt_package_round_trip() const std::filesystem::path plain_path = temp_path("encrypt-plain.xlsx"); const std::filesystem::path encrypted_path = temp_path("encrypt-cipher.xlsx"); - auto written = xl::write_columns(plain_path.string(), XL_FORMAT_XLSX, columns); + auto written = xl::write_columns(plain_path.string(), columns); CHECK(written.has_value(), "write_columns must succeed"); auto encrypted = xl::encrypt_package(plain_path.string(), encrypted_path.string(), "hunter2"); CHECK(encrypted.has_value(), "encrypt_package must succeed"); { - xl::OpenOptions options; - options.password("hunter2"); - auto workbook = xl::Workbook::open(encrypted_path.string(), XL_FORMAT_AUTO, &options); + auto workbook = xl::Workbook::open(encrypted_path.string(), {.password = "hunter2"}); CHECK(workbook.has_value(), "the encrypted file must open with the right password"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the decrypted file must parse back"); CHECK(table->size() == 2, "two rows were written"); WrittenRow first = *table->begin(); @@ -147,7 +146,7 @@ static int test_encrypt_package_rejects_an_empty_password() { const std::filesystem::path plain_path = temp_path("encrypt-empty-pw.xlsx"); const std::array columns{xl::i64_column("inteiro", std::vector{1})}; - auto written = xl::write_columns(plain_path.string(), XL_FORMAT_XLSX, columns); + auto written = xl::write_columns(plain_path.string(), columns); CHECK(written.has_value(), "write_columns must succeed"); auto encrypted = xl::encrypt_package(plain_path.string(), temp_path("encrypt-empty-pw-out.xlsx").string(), ""); @@ -175,13 +174,14 @@ static int test_write_columns_to_memory_round_trip() xl::time_column("hora", horas), xl::timestamp_column("instante", instantes)}; - auto bytes = xl::write_columns_to_memory(XL_FORMAT_XLSX, columns); - CHECK(bytes.has_value(), "write_columns_to_memory must succeed"); - CHECK(!bytes->empty(), "write_columns_to_memory must return non-empty bytes"); + std::vector bytes; + CHECK(xl::write_columns(bytes, columns, {.format = XL_FORMAT_XLSX}).has_value(), + "write_columns into a vector must succeed"); + CHECK(!bytes.empty(), "write_columns must fill the vector"); - auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(bytes, {.format = XL_FORMAT_XLSX}); CHECK(workbook.has_value(), "the returned bytes must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the returned bytes must parse back"); CHECK(table->size() == 2, "two rows were written"); @@ -199,25 +199,29 @@ static int test_write_columns_rejects_bad_input() const std::filesystem::path path = temp_path("rejected.xlsx"); std::array mismatched{xl::i64_column("a", two), xl::i64_column("b", three)}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, mismatched).has_value(), + CHECK(!xl::write_columns(path.string(), mismatched).has_value(), "columns of different lengths must be rejected"); std::array partial_header{xl::i64_column("a", two), xl::i64_column("", two)}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, partial_header).has_value(), + CHECK(!xl::write_columns(path.string(), partial_header).has_value(), "a partial header row must be rejected"); std::array short_bitmap{xl::i64_column("a", two)}; short_bitmap[0].validity = reinterpret_cast(two.data()); short_bitmap[0].validity_len = 0; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, short_bitmap).has_value(), + CHECK(!xl::write_columns(path.string(), short_bitmap).has_value(), "a validity bitmap shorter than the row count must be rejected"); std::array fine{xl::i64_column("a", two)}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_AUTO, fine).has_value(), - "XL_FORMAT_AUTO must be rejected: a new file has no signature bytes to sniff"); + CHECK(!xl::write_columns(temp_path("rejected.txt").string(), fine).has_value(), + "XL_FORMAT_AUTO with an unknown extension must be rejected"); + std::vector untouched{42}; + CHECK(!xl::write_columns(untouched, fine).has_value(), + "XL_FORMAT_AUTO must be rejected for memory: there is no extension to infer it from"); + CHECK(untouched.size() == 1 && untouched[0] == 42, "a failed write leaves the vector untouched"); std::span none{}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, none).has_value(), + CHECK(!xl::write_columns(path.string(), none).has_value(), "an empty column set must be rejected"); std::filesystem::remove(path); @@ -226,35 +230,39 @@ static int test_write_columns_rejects_bad_input() static int test_write_options() { - xl::WriteOptions defaults{}; - xl_write_options raw = defaults.to_c(); - CHECK(raw.struct_size == static_cast(sizeof(xl_write_options)), "to_c must fill struct_size"); - CHECK(raw.sheet_name == nullptr, "an empty sheet_name must lower to NULL, meaning Sheet1"); - CHECK(raw.sheet_name_len == 0, "an empty sheet_name must lower to length 0"); - CHECK(raw.csv_delimiter == 0, "an unset csv_delimiter must lower to 0"); - CHECK(raw.date1904 == XL_OPT_DEFAULT, "an unset tri-state must lower to XL_OPT_DEFAULT"); - - xl::WriteOptions configured{}; - configured.sheet_name = "Dados"; - configured.csv_delimiter = ';'; - configured.use_shared_strings = XL_OPT_TRUE; - xl_write_options set = configured.to_c(); - CHECK(set.sheet_name_len == 5, "sheet_name_len must be the byte length"); - CHECK(set.sheet_name != nullptr, "a non-empty sheet_name must lower to a pointer"); - CHECK(set.csv_delimiter == ';', "csv_delimiter must pass through unchanged"); - CHECK(set.use_shared_strings == XL_OPT_TRUE, "use_shared_strings must pass through unchanged"); + const std::vector values{1, 2}; + const std::array columns{xl::i64_column("a", values), xl::i64_column("b", values)}; + + std::vector bytes; + CHECK(xl::write_columns(bytes, columns, {.format = XL_FORMAT_CSV, .csv_delimiter = ';'}).has_value(), + "writing a CSV into a vector must succeed"); + const std::string_view text(reinterpret_cast(bytes.data()), bytes.size()); + CHECK(text.find("a;b") != std::string_view::npos, "WriteOptions::csv_delimiter must reach the output"); return 0; } -static int test_format_from_path() +static int test_format_from_extension() { - CHECK(xl::format_from_path("out.xlsx") == XL_FORMAT_XLSX, ".xlsx must resolve to XL_FORMAT_XLSX"); - CHECK(xl::format_from_path("out.XLSB") == XL_FORMAT_XLSB, "the extension match must be case-insensitive"); - CHECK(xl::format_from_path("out.xls") == XL_FORMAT_XLS, ".xls must resolve to XL_FORMAT_XLS"); - CHECK(xl::format_from_path("out.csv") == XL_FORMAT_CSV, ".csv must resolve to XL_FORMAT_CSV"); - CHECK(xl::format_from_path("out.txt") == XL_FORMAT_AUTO, "an unknown extension must resolve to AUTO"); - CHECK(xl::format_from_path("out") == XL_FORMAT_AUTO, "no extension at all must resolve to AUTO"); - CHECK(xl::format_from_path("v1.2/report") == XL_FORMAT_AUTO, "a dot before the last separator is not an extension"); + const std::vector values{1, 2}; + const std::array columns{xl::i64_column("a", values)}; + const std::array, 4> cases{{{"ext.xlsx", XL_FORMAT_XLSX}, + {"ext.XLSB", XL_FORMAT_XLSB}, + {"ext.xls", XL_FORMAT_XLS}, + {"ext.csv", XL_FORMAT_CSV}}}; + for (const auto &[name, format] : cases) + { + const std::filesystem::path path = temp_path(name); + CHECK(xl::write_columns(path.string(), columns).has_value(), "a known extension must pick the format"); + { + auto workbook = xl::Workbook::open(path.string(), {.format = format}); + CHECK(workbook.has_value(), "the file must open as the format its extension names"); + auto rows = workbook->sheet(0).read_all(); + CHECK(rows.has_value() && rows->size() == 3, "the header and both rows must read back"); + } + std::filesystem::remove(path); + } + CHECK(!xl::write_columns(temp_path("no-extension").string(), columns).has_value(), + "a path without an extension must be rejected"); return 0; } @@ -278,12 +286,12 @@ static int test_bool_round_trip() const std::array columns{xl::bool_column("ativo", flags)}; const std::filesystem::path path = temp_path("bools.xlsx"); - CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + CHECK(xl::write_columns(path.string(), columns).has_value(), "writing a bool column must succeed"); { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the written file must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 3, "three rows were written"); @@ -316,12 +324,12 @@ static int test_optional_round_trip() const std::array columns{xl::i64_column("quantidade", valores, validity)}; const std::filesystem::path path = temp_path("nullable.xlsx"); - CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + CHECK(xl::write_columns(path.string(), columns).has_value(), "writing a nullable column must succeed"); { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the written file must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 3, "three rows were written"); @@ -381,7 +389,7 @@ static int test_write_sheet_round_trip() { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the written file must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 2, "two rows were written"); @@ -420,12 +428,13 @@ static int test_write_sheet_to_memory_round_trip() std::chrono::system_clock::time_point{std::chrono::microseconds{1767312000000000}}, std::nullopt}}; - auto bytes = xl::write_sheet_to_memory(XL_FORMAT_XLSX, rows); - CHECK(bytes.has_value(), "write_sheet_to_memory must succeed"); + std::vector bytes; + CHECK(xl::write_sheet(bytes, rows, {.format = XL_FORMAT_XLSX}).has_value(), + "write_sheet into a vector must succeed"); - auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(bytes); CHECK(workbook.has_value(), "the returned bytes must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the returned bytes must parse back"); CHECK(table->size() == 2, "two rows were written"); @@ -444,14 +453,12 @@ static int test_write_sheet_options_and_csv() const std::vector rows{}; const std::filesystem::path path = temp_path("named.xlsx"); - xl::WriteOptions options{}; - options.sheet_name = "Dados"; - CHECK(xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows, &options).has_value(), + CHECK(xl::write_sheet(path.string(), rows, {.format = XL_FORMAT_XLSX, .sheet_name = "Dados"}).has_value(), "writing an empty sheet with a custom name must succeed"); { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the written file must open"); - auto name = workbook->sheet_name_at(0); + auto name = workbook->sheet(0).name(); CHECK(name.has_value(), "the sheet name must be readable"); CHECK(*name == "Dados", "WriteOptions::sheet_name must reach the file"); } @@ -509,9 +516,9 @@ static int test_writer_handle() { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the file xl_close_write_handle produced must open"); - auto name = workbook->sheet_name_at(0); + auto name = workbook->sheet(0).name(); CHECK(name.has_value() && *name == "Planilha1", "xl_start_sheet's name must reach the file"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 1, "exactly one data row was written"); WrittenRow first = *table->begin(); @@ -567,7 +574,7 @@ static int test_writer_handle_class_round_trip() { const std::filesystem::path path = temp_path("writer_handle_class.xlsx"); { - auto handle = xl::WriterHandle::open(path.string(), XL_FORMAT_XLSX); + auto handle = xl::WriterHandle::open(path.string()); CHECK(handle.has_value(), "WriterHandle::open must succeed"); CHECK(handle->start_sheet("Planilha1").has_value(), "start_sheet must succeed"); @@ -605,11 +612,13 @@ static int test_writer_handle_class_round_trip() CHECK(handle->end_row().has_value(), "end_row must succeed for the null-cell row"); CHECK(handle->end_sheet().has_value(), "end_sheet must succeed"); + CHECK(handle->finish().has_value(), "finish must succeed"); + CHECK(!handle->start_row().has_value(), "a finished handle accepts no more writes"); } { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the file WriterHandle produced must open"); - auto table = xl::parse_sheet(workbook->sheet(0)); + auto table = xl::parse(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); CHECK(table->size() == 2, "two data rows were written"); @@ -632,44 +641,34 @@ static int test_writer_handle_class_round_trip() static int test_writer_handle_class_to_memory_round_trip() { - auto handle = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); - CHECK(handle.has_value(), "WriterHandle::open_memory must succeed"); + std::vector bytes; + auto handle = xl::WriterHandle::open(bytes, {.format = XL_FORMAT_XLSX}); + CHECK(handle.has_value(), "WriterHandle::open over a vector must succeed"); CHECK(handle->start_sheet("Dados").has_value(), "start_sheet must succeed"); CHECK(handle->start_row().has_value(), "start_row must succeed"); CHECK(handle->write(std::string_view("uma")).has_value(), "write(string_view) must succeed"); CHECK(handle->write(int64_t{3}).has_value(), "write(int64_t) must succeed"); CHECK(handle->end_row().has_value(), "end_row must succeed"); + CHECK(handle->end_sheet().has_value(), "end_sheet must succeed"); + CHECK(bytes.empty(), "the vector is filled only when the handle finishes"); - auto bytes = handle->bytes(); - CHECK(bytes.has_value(), "bytes() must succeed"); - CHECK(!bytes->empty(), "bytes() must return non-empty bytes"); + CHECK(handle->finish().has_value(), "finish must succeed"); + CHECK(!bytes.empty(), "finish must fill the vector"); + CHECK(handle->finish().has_value(), "a second finish does nothing"); - auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); - CHECK(workbook.has_value(), "the bytes bytes() returned must open"); - auto sheet_name = workbook->sheet_name_at(0); + auto workbook = xl::Workbook::open(bytes, {.format = XL_FORMAT_XLSX}); + CHECK(workbook.has_value(), "the bytes finish wrote must open"); + auto sheet_name = workbook->sheet(0).name(); CHECK(sheet_name.has_value() && *sheet_name == "Dados", "start_sheet's name must reach the bytes"); - - auto bytes_again = handle->bytes(); - CHECK(bytes_again.has_value(), "a second bytes() call must still succeed"); - CHECK(*bytes_again == *bytes, "a second bytes() call must return the same content"); return 0; } -static int test_writer_handle_class_bytes_rejects_a_file_backed_handle() +static int test_writer_handle_class_rejects_auto_for_memory() { - const std::filesystem::path path = temp_path("writer_handle_class_file.xlsx"); - { - auto handle = xl::WriterHandle::open(path.string(), XL_FORMAT_XLSX); - CHECK(handle.has_value(), "WriterHandle::open must succeed"); - CHECK(handle->start_sheet("S").has_value(), "start_sheet must succeed"); - - auto bytes = handle->bytes(); - CHECK(!bytes.has_value(), "bytes() on a file-backed handle must fail"); - CHECK(bytes.error().code == XL_INVALID_ARGUMENT, - "bytes() on a file-backed handle must be XL_INVALID_ARGUMENT"); - } - std::filesystem::remove(path); + std::vector bytes; + auto handle = xl::WriterHandle::open(bytes); + CHECK(!handle.has_value(), "WriterHandle::open over a vector must reject XL_FORMAT_AUTO"); return 0; } @@ -679,7 +678,7 @@ int main() { return 1; } - if (test_format_from_path() != 0) + if (test_format_from_extension() != 0) { return 1; } @@ -743,7 +742,7 @@ int main() { return 1; } - if (test_writer_handle_class_bytes_rejects_a_file_backed_handle() != 0) + if (test_writer_handle_class_rejects_auto_for_memory() != 0) { return 1; } diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md index 555605d6..3c4eca89 100644 --- a/docs/guide/migrating-to-v7.md +++ b/docs/guide/migrating-to-v7.md @@ -160,3 +160,123 @@ The C ABI is unchanged: no signature moved, and `XL_ABI_VERSION` stays 6. Behavi `xl_write_options.date1904` now applies to XLSX as well as XLSB and XLS; v6 ignored it for XLSX. `xl_csv_aggregate` keeps its signature. Inside, it opens a [`CsvReader`](xref:ExcelReader.Core.Reader.Csv.CsvReader) with the dialect from the native struct and calls `SheetParallel`. + +### Python: one way to open a workbook + +`open_bytes()` is removed. `open_workbook()` takes the bytes instead, and also accepts a `bytearray`, a `memoryview` or an `io.BytesIO`. It also takes a `Source` or a readable stream, the inputs that 7.0's release candidates opened with `open_source()` and `open_stream()`, which are gone as well. Arguments and keywords are unchanged. + +| v6 | v7 | +|---|---| +| `open_bytes(data, "xlsx", password=p)` | `open_workbook(data, "xlsx", password=p)` | +| `open_bytes(buffer.getvalue())` | `open_workbook(buffer)` | + +### Python: one way to write a workbook + +`write_workbook_to_bytes()`, `write_arrow_to_bytes()`, `write_pandas_to_bytes()` and `write_polars_to_bytes()` are removed. `write_workbook()`, `write_arrow()`, `write_pandas()` and `write_polars()` take the destination first, a path or a binary object with `write()`, such as `io.BytesIO`. A destination without a path needs `format`. The frame writers now take `format` and `options` as keywords instead of passing `**kwargs` through. + +| v6 | v7 | +|---|---| +| `payload = write_workbook_to_bytes(table, types, format="xlsx")` | `buffer = io.BytesIO()`
`write_workbook(buffer, table, types, format="xlsx")`
`payload = buffer.getvalue()` | +| `write_pandas_to_bytes(df, format="xlsx")` | `write_pandas(buffer, df, format="xlsx")` | + +The streaming writer and the encryption helper follow the same pattern. `open_writer_to_memory()` and `SheetWriter.bytes()` are removed: `open_writer()` takes a binary object with `write()` and hands it the finished workbook when the writer closes. `encrypt_package_bytes()` is removed: `encrypt_package()` takes the package as a path, `bytes` or `io.BytesIO`, and the destination as a path or an object with `write()`. + +| v6 | v7 | +|---|---| +| `with open_writer_to_memory("xlsx") as w:`
` ...`
` payload = w.bytes()` | `buffer = io.BytesIO()`
`with open_writer(buffer, "xlsx") as w:`
` ...`
`payload = buffer.getvalue()` | +| `secret = encrypt_package_bytes(payload, password)` | `encrypt_package(payload, out, password)` with `out = io.BytesIO()` | + +### Python: one shape per read + +A sheet reads as rows (`rows()`), a typed table (`parse_typed()`, or `iter_parse_typed()` in batches), Arrow (`to_arrow()`, or `to_arrow_stream()` in batches) or a DataFrame (`to_pandas()`, `to_polars()`). `write_workbook()` takes a `TypedTable` or any Arrow-compatible table, including pandas and polars DataFrames. `SheetWriter` writes whole rows only. + +| v6 | v7 | +|---|---| +| `workbook.sheet_count` | `len(workbook.sheets)` | +| `workbook.sheet_names` | `[s.name for s in workbook.sheets]` | +| `workbook.sheet_name_at(i)` | `workbook.sheets[i].name` | +| `sheet.read_all()` | `list(sheet.rows())`, or `parse_typed()` for speed | +| `sheet.read_all_columnar()`, `ColumnarSheet`, `decode_cell()` | `sheet.parse_typed(sheet.infer_schema())` | +| `sheet.to_arrow()` returning a `StructArray` | `sheet.to_arrow()` returns a `pyarrow.RecordBatch` | +| `sheet.to_record_batch(...)` | `sheet.to_arrow(...)` | +| `sheet.to_record_batch_reader(...)` | `pyarrow.RecordBatchReader.from_stream(sheet.to_arrow_stream(...))` | +| `sheet.iter_pandas(...)`, `sheet.iter_polars(...)` | iterate the reader above and convert each batch | +| `write_arrow(dest, batch)`, `write_pandas(dest, df)`, `write_polars(dest, df)` | `write_workbook(dest, batch_or_df)` | +| `writer.start_row()`, `write_str(v)` … `write_timestamp(v)`, `end_row()` | `writer.write_row([...])` | +| `writer.write_null(ColumnType.X)` | `(None, ColumnType.X)` as a value in `write_row` | +| `OpenOptions(csv_max_cell_bytes=n)`, `OpenOptions(csv_intern_strings=b)` | `OpenOptions(max_cell_bytes=n)`, `OpenOptions(intern_strings=b)`, which now apply to CSV too | +| `from excelreader import ArrowStream` | `from excelreader.arrow_stream import ArrowStream` | + +### Python: value types are not tuples + +`Cell`, `ColumnSpec`, `OpenOptions`, `WriteOptions` and `TypedTable` are frozen dataclasses instead of named tuples, so a field added in a later version breaks no caller. They cannot be unpacked or indexed, and fields are passed by name; `ColumnSpec` keeps `type` as its one positional argument. `open_writer()` takes `options` as a keyword. Python 3.10 or later is required. + +| v6 | v7 | +|---|---| +| `column, type_, value = cell` | `cell.column`, `cell.type`, `cell.value` | +| `ColumnSpec(ColumnType.I64, "qty")` | `ColumnSpec(ColumnType.I64, name="qty")` | +| `open_writer(dest, "xlsx", opts)` | `open_writer(dest, "xlsx", options=opts)` | + +### C++: one overload set per operation + +`Workbook::open` takes a path, a byte span, a `Source` or an `InputStream`. `write_columns`, `write_sheet` and `WriterHandle::open` take a path or a `std::vector&`. Options are passed as `const Options& = {}`, and the format moved into `OpenOptions::format` and `WriteOptions::format` (`XL_FORMAT_AUTO` infers it from a path's extension). + +| v6 | v7 | +|---|---| +| `Workbook::open(path, format, &opts)` | `Workbook::open(path, {.format = format, ...})` | +| `Workbook::open_memory(span, ...)`, `open_source(...)`, `open_stream(...)` | `Workbook::open(span | source | stream, opts)` | +| `sheet_count()`, `sheet_name_at(i)`, `sheet_names()` | `sheets()`, `sheet(i).name()` | +| `Sheet::read_all_decoded()`, `RowCursor::read_all_decoded()` | `Sheet::read_all()` | +| `typed_reader` / `arrow_stream` with `batch_size = 0` | `parse_sheet` / `parse_arrow` | +| `TableView::to_vector()` | `std::vector(t.begin(), t.end())` | +| `handle()`, `from_raw`, constructors from C handles, `to_c()` | removed or private | +| `XlType`, `FieldBinding` | `xl::detail` | +| `OpenOptions::password(sv)` | `OpenOptions::password` field | +| `OpenOptions::csv_max_cell_bytes`, `csv_intern_strings` | `max_cell_bytes`, `intern_strings`, which now apply to CSV too | +| `write_columns_to_memory(format, cols, &opts)` | `write_columns(out_vector, cols, {.format = format})` | +| `write_sheet_to_memory(format, rows, &opts)` | `write_sheet(out_vector, rows, {.format = format})` | +| `format_from_path(path)` | internal | +| `WriterHandle::open_memory(format, &opts)` + `bytes()` | `WriterHandle::open(out_vector, {.format = format})` + `finish()` | +| `aggregate_csv_file` / `aggregate_csv_memory` with `CsvParallelOptions` | `aggregate_csv(path | span, seed, opts, header_row, dop)` | +| `ArrowSchemaGuard`, `ArrowArrayGuard` | `ArrowTable` | + +### Rust: one path per item, no raw FFI + +Every public item lives at the crate root. The raw C interface (`Xl*`, `xl_*`, `XL_*`) is private, and integer codes became the enums `Format`, `ColumnType` and `ErrorKind`. `OpenOptions` and `WriteOptions` are builder-only and carry the format. + +| v6 | v7 | +|---|---| +| `excelreader::workbook::*`, `writer::*`, `rows::*`, … | `excelreader::*` | +| `XL_FORMAT_*`, `XL_T_*`, `Error::code()` | `Format`, `ColumnType`, `Error::kind()` | +| `Workbook::open_with(path, fmt, Some(&o))` | `o.format(fmt).open(path)` | +| `Workbook::open_memory` / `open_source` / `open_reader` | `OpenOptions::open_bytes` / `open_source` / `open_reader` | +| `sheet_count` / `sheet_names` / `sheet_name_at` | `sheets()`, `Sheet::name()` | +| `parse_sheet` / `parse_sheet_parallel` | `sheet.parse::(header_row, parallelism)` | +| `parse_arrow` / `parse_arrow_parallel` / `parse_arrow_stream` | `sheet.parse_arrow::(…)` / `sheet.parse_arrow_stream::(…)` | +| `read_all_decoded` / `read_all_blob` | `read_all()` | +| `infer_schema_parse_text(h, n)` | `infer_schema(h, n, true)` | +| `write_columns(path, fmt, cols, opts)` | `write_columns(path, cols, &opts)`, format in `WriteOptions` | +| `write_columns_to_memory` / `write_sheet_to_memory` | `write_columns_to(writer, …)` / `write_sheet_to(writer, …)` | +| `WriterHandle::open` / `open_with` | `WriterHandle::create(path, &opts)` | +| `WriterHandle::open_memory` + `bytes()` | `WriterHandle::new(writer, &opts)` + `finish()` | +| `WriterHandle::write_null(t)` | `write_*(None)` | +| `aggregate_csv_file` / `aggregate_csv_memory`, `CsvParallelOptions` | `aggregate_csv(input, &opts, header_row, parallelism, seed)` | +| `OpenOptions::csv_max_cell_bytes` / `csv_intern_strings` | `max_cell_bytes` / `intern_strings`, which now apply to CSV too | +| `Column { name, data, validity: None }` | `Column::new(name, data)` | +| `Column { name, data, validity: Some(bitmap) }` | `Column::new(name, data).validity(bitmap)` | + +`CellType`, `ColumnData`, `CsvInput`, `SheetVisibility`, `Column` and `InferredColumn` are `#[non_exhaustive]`, so a later version can add a variant or a field without a major release. A `match` on one of the enums needs a `_` arm, and a `Column` is built with `Column::new`. + +### One vocabulary across the bindings + +Python, C++ and Rust read a sheet with the same four names: `parse` (typed, whole sheet), `parse_batches` (typed, in batches), `to_arrow` (Arrow, whole sheet) and `to_arrow_stream` (Arrow, in batches). The header row, parallelism and batch size travel together: as keywords in Python, a `ReadOptions` aggregate in C++ and a `ReadOptions` builder in Rust. + +| Binding | v7 release candidates | v7 | +|---|---|---| +| Python | `sheet.parse_typed(schema, 1, 0)`, `sheet.iter_parse_typed(schema, 1, 500)` | `sheet.parse(schema, parallelism=0)`, `sheet.parse_batches(schema, batch_size=500)`; every read option is keyword-only | +| Python | `cell.as_date(workbook.is_date1904)` | `cell.as_date()`; the cell carries `date1904` | +| C++ | `parse_sheet(s, h, n)`, `typed_reader(s, h, b)` | `parse(s, {.header_row = h, .parallelism = n})`, `parse_batches(s, {.batch_size = b})` | +| C++ | `parse_arrow(...)`, `arrow_stream(...)` | `to_arrow(s, opts)`, `to_arrow_stream(s, opts)` | +| Rust | `sheet.parse::(h, n)`, `sheet.typed_chunks::(h, b)` | `sheet.parse::(&ReadOptions::new().header_row(h).parallelism(n))`, `sheet.parse_batches::(&opts)` | +| Rust | `sheet.parse_arrow::(...)`, `sheet.parse_arrow_stream::(...)`, `TypedChunks`, `ArrowChunks` | `sheet.to_arrow::(&opts)`, `sheet.to_arrow_stream::(&opts)`, `TypedBatches`, `ArrowBatches` | +| Rust | `Date::new(d)`, `Time::new(t)`, `Timestamp::new(ts)` | `Date { days_since_epoch: d }`, `Time { micros_since_midnight: t }`, `Timestamp { micros_since_epoch: ts }` | diff --git a/docs/guide/versioning.md b/docs/guide/versioning.md new file mode 100644 index 00000000..a4a15214 --- /dev/null +++ b/docs/guide/versioning.md @@ -0,0 +1,86 @@ +# Versioning and compatibility + +What a new version can change, and how to write code that keeps compiling and running across minor +versions. The .NET packages follow [Semantic Versioning](https://semver.org/): a major version (7.0, +8.0) may break your code, while a minor (7.1) or a patch (7.0.1) does not. + +## What a minor version may add + +- New types, new members on existing types, and new overloads. +- New properties on the options objects, such as + [`ExcelReaderOptions`](xref:ExcelReader.Core.Reader.ExcelReaderOptions). A new option always defaults + to the previous behavior. +- New members on the interfaces that only this library implements (below). +- New values in the enums (below). +- Bug fixes. A file that used to read wrong, or a written file that Excel refused or altered, can come + out differently after a fix. Other changes in what an existing call returns or throws wait for a + major version and are listed in its migration guide. + +A minor version does not remove or rename anything, change an existing signature, or add a +parameter to an existing method. + +## Interfaces you implement + +These are the extension points. A minor version never adds a member to them that you would have to +implement: + +- [`IExcelCellConverter`](xref:ExcelReader.Core.Parser.IExcelCellConverter`1) and + [`IExcelCellWriter`](xref:ExcelReader.Core.Writer.IExcelCellWriter`1) +- [`IExcelRowMap`](xref:ExcelReader.Core.Parser.IExcelRowMap`1) and + [`IExcelRecordMap`](xref:ExcelReader.Core.Writer.IExcelRecordMap`1) +- [`IRowRecord`](xref:ExcelReader.Core.Reader.IRowRecord`1) and + [`IRowAccumulator`](xref:ExcelReader.Core.Reader.IRowAccumulator`2) + +New information reaches them through the types they already receive. For example, a converter gets +it as a new member of [`ExcelRowContext`](xref:ExcelReader.Core.Parser.ExcelRowContext), not as a +new parameter. + +## Interfaces only the library implements + +The reading and writing interfaces describe what every format's workbook, sheet and row writer can +do. Program against them, but do not implement them: + +- [`IExcelWorkbook`](xref:ExcelReader.Core.Reader.IExcelWorkbook), + [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet) and + [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet`1) +- [`IExcelRowEnumerator`](xref:ExcelReader.Core.Reader.IExcelRowEnumerator) +- [`IWorkbookWriter`](xref:ExcelReader.Core.Writer.IWorkbookWriter`1), + [`ISheetWriter`](xref:ExcelReader.Core.Writer.ISheetWriter`1), + [`ITableSheetWriter`](xref:ExcelReader.Core.Writer.ITableSheetWriter`1) and + [`IRowWriter`](xref:ExcelReader.Core.Writer.IRowWriter) + +A minor version can add members to them. Each new member comes with a default implementation, so an +existing implementation still compiles and loads, but it throws `NotSupportedException` or falls back +to a slower path for the new member. A feature that only some formats support appears first on the +concrete types (`XlsxSheetWriter`, `XlsbWorkbook`), or as a separate interface the way +[`ITableSheetWriter`](xref:ExcelReader.Core.Writer.ITableSheetWriter`1) sits beside +[`ISheetWriter`](xref:ExcelReader.Core.Writer.ISheetWriter`1). + +## Enums that grow + +[`CellType`](xref:ExcelReader.Core.Reader.CellType) and +[`ExcelColumnType`](xref:ExcelReader.Core.Reader.Schema.ExcelColumnType) can gain values in a minor +version, as can their native counterparts `XL_CELL_*` and `XL_T_*`. An existing value never changes +its number or meaning. A `switch` over them should have a default branch: + + +```csharp +string text = cell.Type switch +{ + CellType.Empty => "", + CellType.Number when cell.TryGetDouble(out double number) => number.ToString(CultureInfo.InvariantCulture), + _ => cell.GetString(), +}; +``` + +## The native library and the bindings + +The C ABI is versioned on its own by `XL_ABI_VERSION`, which `xl_abi_version()` reports. It changes +only when a signature or a struct layout does: 7.0 kept version 6. Every options struct starts with +`struct_size`, so a field added in a later version does not break a caller compiled against an +earlier header. The Rust, Python and C++ bindings check the ABI version when they load the library. + +## How it is enforced + +Every pull request packs `ExcelReader.NET` and `ExcelReader.Arrow` and compares them with the last +release through .NET package validation. A removed or changed member fails the build. diff --git a/docs/snippets/Versioning.cs b/docs/snippets/Versioning.cs new file mode 100644 index 00000000..76c6af3a --- /dev/null +++ b/docs/snippets/Versioning.cs @@ -0,0 +1,20 @@ +using System.Globalization; + +namespace Snippets.Versioning.EnumSwitch +{ + internal static class Snippet + { + internal static string Run(Cell cell) + { + #region versioning-enum-switch + string text = cell.Type switch + { + CellType.Empty => "", + CellType.Number when cell.TryGetDouble(out double number) => number.ToString(CultureInfo.InvariantCulture), + _ => cell.GetString(), + }; + #endregion + return text; + } + } +} diff --git a/docs/toc.yml b/docs/toc.yml index d3707a7d..62aad411 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -14,6 +14,8 @@ href: guide/errors.md - name: Reading untrusted files href: guide/untrusted-input.md + - name: Versioning and compatibility + href: guide/versioning.md - name: API href: api/ - name: Bindings diff --git a/python/README.md b/python/README.md index d694c42f..4463ffa5 100644 --- a/python/README.md +++ b/python/README.md @@ -22,7 +22,7 @@ elsewhere, set `EXCELREADER_NATIVE_LIB` to its full path. from excelreader import open_workbook with open_workbook("book.xlsx") as workbook: - print(workbook.sheet_count, workbook.sheets[0].name) + print(len(workbook.sheets), workbook.sheets[0].name) for row in workbook.sheets[0].rows(): for cell in row: print(cell.column, cell.type.name, cell.value) @@ -57,9 +57,9 @@ with excelreader.open_workbook("report.xlsx") as workbook, ThreadPoolExecutor() tables = list(pool.map(lambda sheet: sheet.to_arrow(sheet.infer_schema()), workbook.sheets)) ``` -`rows()` and `iter_parse_typed()` are generators: the native cursor opens at the first `next()`. A +`rows()` and `parse_batches()` are generators: the native cursor opens at the first `next()`. A generator that has already started keeps working after `workbook.close()`; one that has not started -raises at its first `next()`. A `RecordBatchReader` opens its stream at the call, so it works after +raises at its first `next()`. `to_arrow_stream()` opens its stream at the call, so it works after `close()` even if never started. ### Formats @@ -82,53 +82,34 @@ as_date = cell.as_date(workbook.is_date1904) `as_date()` returns `None` for any cell that isn't `CellType.DATE`. -### Reading everything at once +### Reading a sheet -`rows()` iterates row-by-row; `read_all()` materializes the whole sheet in one call: +A sheet reads in one of these shapes: -```python -all_rows = workbook.sheets[0].read_all() -``` - -This holds every row in memory at once, so prefer `rows()` for very large sheets. - -### Reading everything at once, faster +| Method | Returns | Knob | Needs | +|---|---|---|---| +| `rows()` | one list of `Cell` per row, streamed | — | nothing | +| `parse()` | the whole sheet as a `TypedTable` | `parallelism` | nothing | +| `parse_batches()` | one `TypedTable` per batch | `batch_size` | nothing | +| `to_arrow()` | the whole sheet as a `pyarrow.RecordBatch` | `parallelism` | pyarrow | +| `to_arrow_stream()` | an Arrow PyCapsule stream of batches | `batch_size` | nothing | +| `to_pandas()` / `to_polars()` | a DataFrame | `batch_size`, `parallelism` | pandas + pyarrow / polars | -`read_all()`/`rows()` build one `Cell`/`str` object per cell, which dominates wall-clock time on a -large sheet. `read_all_columnar()` decodes the same data into parallel flat arrays instead — no -per-cell object construction — and is several times faster on large sheets: - -```python -columnar = workbook.sheets[0].read_all_columnar() -# columnar.row_offsets[i]:row_offsets[i+1] -> cell indices for row i -# columnar.columns[j] / columnar.types[j] -> cell j's column index / CellType -# columnar.value_offsets[j]:[j+1] -> cell j's byte slice into columnar.values -``` - -Materialize a single cell on demand instead of decoding every value up front: - -```python -from excelreader import decode_cell - -first_cell = decode_cell(columnar, 0) -``` - -Each array is a stdlib `array.array('i')`, or a NumPy `int32` array if NumPy is installed -(`pip install -e "python[numpy]"`) — NumPy is optional and never required. +`infer_schema()` guesses the schema every typed method takes. ### Typed columns — the fastest path -Everything above hands back cell *text*, which means the library formats every value to a string on -the way out. `parse_typed()` skips that entirely: you give it a schema, and the conversion happens -natively, straight into typed column buffers. On a 65,536 × 14 sheet it is ~8× faster than -`read_all_columnar()`, ~25× faster than `read_all()`, and faster than `polars.read_excel()` — see -[Reading](#reading) below for the measured numbers. +`rows()` hands back cell *text*, which means the library formats every value to a string on the way +out and builds one Python object per cell. `parse()` skips that entirely: you give it a schema, +and the conversion happens natively, straight into typed column buffers. On a 65,536 × 14 sheet it is +~30× faster than `rows()`, and faster than `polars.read_excel()` — see [Reading](#reading) below for +the measured numbers. ```python from excelreader import ColumnSpec, ColumnType with open_workbook("sales.xlsb") as workbook: - table = workbook.sheets[0].parse_typed([ + table = workbook.sheets[0].parse([ ColumnSpec(ColumnType.STRING, name="Region"), ColumnSpec(ColumnType.DATE, name="Order Date"), ColumnSpec(ColumnType.F64, name="Total Revenue", nullable=True), @@ -143,6 +124,9 @@ revenue[0] # 14862.69 table.validity[2] # bit-packed nulls, or None when the column has none ``` +Each non-string column is a stdlib `array.array`, or a NumPy array if NumPy is installed +(`pip install -e "python[numpy]"`) — NumPy is optional and never required. + Leave `name` out to resolve a column by position instead: `ColumnSpec(ColumnType.I64, index=3)`. `header_row` defaults to 1 (the first row names the columns); pass `header_row=0` for a sheet with no header, where every spec must resolve by index. @@ -150,12 +134,12 @@ header, where every spec must resolve by index. A column that fails to convert is an error unless its spec sets `nullable=True`, which records the failure in `table.validity` and keeps reading. -`parse_typed()` always reads the whole sheet from its first row, independent of any `rows()` +`parse()` always reads the whole sheet from its first row, independent of any `rows()` generator in progress. #### Reading a sheet a batch at a time -For a sheet too large to hold in memory at once, `iter_parse_typed()` is `parse_typed()` a batch at a +For a sheet too large to hold in memory at once, `parse_batches()` is `parse()` a batch at a time — a generator yielding one `TypedTable` per batch: ```python @@ -165,26 +149,26 @@ with open_workbook("sales.xlsb") as workbook: ColumnSpec(ColumnType.DATE, name="Order Date"), ColumnSpec(ColumnType.F64, name="Total Revenue", nullable=True), ] - for batch in workbook.sheets[0].iter_parse_typed(schema, batch_size=10_000): + for batch in workbook.sheets[0].parse_batches(schema, batch_size=10_000): region, day, revenue = batch.columns ... ``` -`batch_size` is rows per batch; `0` means one unbounded batch, identical to `parse_typed()`, and a +`batch_size` is rows per batch; `0` means one unbounded batch, identical to `parse()`, and a negative value is an error. Chunked reads are independent: several can be open on one workbook at once, typed or Arrow (see [Arrow](#arrow) below), and none disturbs another. -Being a generator, `iter_parse_typed()` opens nothing until the first iteration, so a bad +Being a generator, `parse_batches()` opens nothing until the first iteration, so a bad `batch_size` is only raised there, not at the call. #### Reading a large sheet on several threads -`parse_typed()`, `to_arrow()`, `to_record_batch()` and `to_polars()` take `parallelism`: `1` (the +`parse()`, `to_arrow()`, `to_pandas()` and `to_polars()` take `parallelism`: `1` (the default) reads on one thread, `0` uses every core, `n` up to n threads. XLSX, XLSB and CSV are split. XLS, or a sheet under 256 KiB, is read on one thread with the same result. The table is identical either way. Every partition's columns are held until they are merged, so the peak memory is higher -than a sequential read of the same file. `to_polars()` with `parallelism` other than 1 parses the whole -file instead of streaming it in batches. +than a sequential read of the same file. `to_pandas()` and `to_polars()` with `parallelism` other +than 1 parse the whole file instead of streaming it in batches. `xl_parse_typed` on a Ryzen 7 5700X (8 cores, 16 threads), 14 typed columns, file read from memory: @@ -210,7 +194,7 @@ don't, `infer_schema()` samples the sheet and guesses one for you: with open_workbook("sales.xlsb") as workbook: sheet = workbook.sheets[0] schema = sheet.infer_schema() # header_row=1, sample_size=100 by default - table = sheet.parse_typed(schema) + table = sheet.parse(schema) ``` By default each column's type comes from the `CellType` Excel already stored for its sampled cells, @@ -227,12 +211,12 @@ still a guess over the sample, so a value further down can fail to convert: with open_workbook("sales.csv") as workbook: sheet = workbook.sheets[0] schema = sheet.infer_schema(parse_text=True) - table = sheet.parse_typed(schema, parallelism=0) + table = sheet.parse(schema, parallelism=0) ``` ### Writing -`write_workbook()` writes a `TypedTable` (what `parse_typed()` returns) back out as a single sheet, +`write_workbook()` writes a `TypedTable` (what `parse()` returns) back out as a single sheet, through the same `xl_write_typed` native export — one-shot, no writer handle before or after the call: ```python @@ -240,7 +224,7 @@ from excelreader import ColumnType, write_workbook with open_workbook("sales.xlsb") as workbook: sheet = workbook.sheets[0] - table = sheet.parse_typed(sheet.infer_schema()) + table = sheet.parse(sheet.infer_schema()) types = [ColumnType.STRING, ColumnType.DATE, ColumnType.F64] # one per table.columns, in order write_workbook("sales_copy.xlsx", table, types) @@ -254,14 +238,27 @@ arrays. `format` is inferred from the path's extension (one of xlsx/xlsb/xls/csv write_workbook("report.dat", table, types, format="csv") ``` -`write_pandas()` and `write_polars()` build the table from a DataFrame instead (both go through -`write_arrow()`, so `pyarrow` must be installed): +The destination can also be a binary object with `write()`, such as `io.BytesIO` or an open file. The +workbook is built in memory and written in one `write()` call, and with no extension to infer it from, +`format` is required: ```python -from excelreader import write_pandas, write_polars +import io + +buffer = io.BytesIO() +write_workbook(buffer, table, types, format="xlsx") +payload = buffer.getvalue() +``` + +`write_workbook()` also takes Arrow-shaped data in place of a `TypedTable`: a `pyarrow.RecordBatch` +or `Table`, a pandas DataFrame (its index is not written), or any table exposing +`__arrow_c_stream__`, such as a polars DataFrame. Each column's type comes from its Arrow type, so +leave `types` out. This path requires `pyarrow`: -write_pandas("report.xlsx", df) # requires pandas + pyarrow -write_polars("report.xlsx", polars_df) # requires polars + pyarrow +```python +write_workbook("report.xlsx", df) # pandas +write_workbook("report.xlsx", polars_df) # polars +write_workbook("report.xlsx", record_batch) ``` `WriteOptions` sets the sheet name and CSV dialect, mirroring `xl_write_options` — every field @@ -297,80 +294,68 @@ with open_writer("out.xlsx") as writer: writer.end_sheet() ``` -`write_row` picks each cell's type from the Python value. For control over a column's type — or to -write a typed blank — use the explicit methods: `write_str`, `write_i64`, `write_f64`, `write_bool`, -`write_date`, `write_time`, `write_timestamp`, and `write_null(ColumnType.I64)`. A `None` passed to -`write_row` becomes a blank string cell. - -To build a workbook without touching the filesystem, use `open_writer_to_memory` and read the result -out with `bytes()`: +`write_row` picks each cell's type from the Python value: `str`, `bool`, `int` (64-bit), `float`, +`datetime.datetime` (an aware one is converted to UTC), `datetime.date` and `datetime.time`. A `None` +becomes a blank string cell; the tuple `(None, ColumnType.I64)` writes a blank cell of that type: ```python -from excelreader import open_writer_to_memory - -with open_writer_to_memory("xlsx") as writer: - writer.start_sheet("Data") - writer.write_row(["name", "qty"]) - writer.end_sheet() - payload = writer.bytes() +writer.write_row(["widget", (None, ColumnType.I64), 2.5]) ``` -`write_workbook_to_bytes()` is the same idea for the columnar `write_workbook()` path, and -`write_arrow_to_bytes()`, `write_pandas_to_bytes()` and `write_polars_to_bytes()` for the frame -writers. With no path to infer it from, each takes an explicit `format`: +To build a workbook without touching the filesystem, open the writer over a binary object with +`write()`, such as `io.BytesIO`. It needs `format`, and receives the finished workbook in one +`write()` call when the writer closes: ```python -from excelreader import write_pandas_to_bytes +import io -payload = write_pandas_to_bytes(df, format="xlsx") +from excelreader import open_writer + +buffer = io.BytesIO() +with open_writer(buffer, "xlsx") as writer: + writer.start_sheet("Data") + writer.write_row(["name", "qty"]) + writer.end_sheet() +payload = buffer.getvalue() ``` ### Arrow With `pyarrow` installed, `to_arrow()` runs the same read and hands the buffers to pyarrow zero-copy -over the Arrow C Data Interface: +over the Arrow C Data Interface, as one column-named `pyarrow.RecordBatch`: ```python -import pyarrow as pa - with open_workbook("sales.xlsb") as workbook: - array = workbook.sheets[0].to_arrow(schema) - -batch = pa.RecordBatch.from_struct_array(array) + batch = workbook.sheets[0].to_arrow(schema) ``` pyarrow owns the buffers from that point on, so the result stays valid after the workbook is closed. -#### Streaming: RecordBatchReader, pandas, and polars +#### Streaming: Arrow streams, pandas, and polars -`to_record_batch_reader()` is `to_arrow()` a batch at a time, as a streaming -`pyarrow.RecordBatchReader` — peak memory is one batch rather than one sheet: +`to_arrow_stream()` is `to_arrow()` a batch at a time — peak memory is one batch rather than one +sheet. It returns a single-use stream implementing the Arrow PyCapsule interface +(`__arrow_c_stream__`) and needs no third-party package itself; hand it to any consumer, such as +pyarrow for a `RecordBatchReader`: ```python +import pyarrow as pa + with open_workbook("sales.xlsb") as workbook: - reader = workbook.sheets[0].to_record_batch_reader(schema, batch_size=10_000) + reader = pa.RecordBatchReader.from_stream(workbook.sheets[0].to_arrow_stream(schema, batch_size=10_000)) for batch in reader: - ... + frame = batch.to_pandas() # or polars.from_arrow(batch), for one DataFrame per batch ``` -Same `batch_size` rules as `iter_parse_typed()` above, except `to_record_batch_reader()` raises -immediately rather than on the first iteration. The reader keeps working after the workbook is closed. - -`iter_pandas()`/`iter_polars()` build on it, yielding one DataFrame per batch (requires pyarrow, plus -pandas or polars respectively): +Same `batch_size` rules as `parse_batches()` above, except `to_arrow_stream()` raises +immediately rather than on the first iteration. The stream keeps working after the workbook is closed. -```python -with open_workbook("sales.xlsb") as workbook: - for frame in workbook.sheets[0].iter_polars(schema, batch_size=10_000): - ... -``` - -`to_pandas()`/`to_polars()` materialize the whole sheet as a single DataFrame and also take a -`batch_size`, but the two spend it differently — do not blur them. `to_polars()` genuinely streams: -polars consumes the reader batch by batch, so the whole sheet is never resident as Arrow buffers at -once. `to_pandas()` does **not** bound peak memory the same way — `RecordBatchReader.read_all()` -concatenates every batch before pandas ever sees them. What `batch_size` buys `to_pandas()` instead is -the conversion: `self_destruct=True` frees each Arrow chunk as pandas consumes it, so the sheet is +`to_pandas()`/`to_polars()` materialize the whole sheet as a single DataFrame and both take +`batch_size` and `parallelism`, but they spend `batch_size` differently — do not blur them. +`to_polars()` genuinely streams: polars consumes the stream batch by batch, so the whole sheet is +never resident as Arrow buffers at once. `to_pandas()` does **not** bound peak memory the same way — +it concatenates every batch before pandas ever sees them. What `batch_size` buys `to_pandas()` instead +is the conversion: `self_destruct=True` frees each Arrow chunk as pandas consumes it, so the sheet is never held twice, once as Arrow and once as the DataFrame. **A faulted stream surfaces differently from every other native error in this library**: pyarrow's @@ -378,25 +363,25 @@ never held twice, once as Arrow and once as the DataFrame. `ExcelReaderError` — the failure crosses the Arrow C Data Interface before this library's own exception wrapping ever runs. -### From memory +### From memory, a stream, or your own bytes -```python -from excelreader import open_bytes +`open_workbook()` takes more than a path. `bytes`, `bytearray`, `memoryview` and `io.BytesIO` are +read from memory; the format comes from the content, so pass `format` only for a CSV or to skip the +check. -with open_bytes(payload) as workbook: +```python +with open_workbook(payload) as workbook: ... ``` -### From your own bytes - -`open_source()` takes any object with a `size` and a `read_at(offset, buffer)` that fills `buffer` -and returns the count. The library asks for 4 MiB blocks and caches them, and every sheet can still -be read on its own thread. +An object with a `size` and a `read_at(offset, buffer)` that fills `buffer` and returns the count is +read in blocks on demand: the library asks for 4 MiB blocks and caches them, and every sheet can +still be read on its own thread. ```python import urllib.request -from excelreader import open_source +from excelreader import open_workbook class HttpSource: def __init__(self, url): @@ -411,28 +396,28 @@ class HttpSource: buffer[: len(data)] = data return len(data) -with open_source(HttpSource("https://example.com/book.xlsx")) as workbook: +with open_workbook(HttpSource("https://example.com/book.xlsx")) as workbook: ... ``` -`open_stream()` reads anything with `readinto()` or `read()`, such as `sys.stdin.buffer`. A CSV is -read as it arrives, and its first 16 MiB are kept so that `infer_schema()` followed by a read works; +Any other object with `readinto()` or `read()`, such as an open file or `sys.stdin.buffer`, is read +as a stream. A CSV is read as it arrives, and its first 16 MiB are kept so that `infer_schema()` followed by a read works; an XLSX, XLSB or XLS is read whole first. ```python import sys -from excelreader import open_stream +from excelreader import open_workbook -with open_stream(sys.stdin.buffer, "csv") as workbook: +with open_workbook(sys.stdin.buffer, "csv") as workbook: ... ``` -Both callbacks run with the GIL held. For a file on disk, `open_workbook()` is faster. +Both callbacks run with the GIL held. For a file on disk, pass its path: that is faster. ### Reader options -`open_workbook()`/`open_bytes()` take an optional `OpenOptions` for CSV dialect settings and reader +`open_workbook()` takes an optional `OpenOptions` for CSV dialect settings and reader resource limits. Every field defaults to `None`, meaning "use the library default", so you set only what you want to change. @@ -446,6 +431,7 @@ with open_workbook("export.csv", format="csv", options=OpenOptions(csv_delimiter ``` `csv_delimiter` and `csv_quote` are byte values, so pass `ord(";")` rather than `";"`. +`max_cell_bytes` and `intern_strings` apply to every format, a CSV included. The `max_*` fields are resource limits rather than tuning knobs: they bound what a malformed or hostile file can make the reader allocate, and exceeding one raises `ExcelReaderError`. Lower them @@ -469,7 +455,7 @@ check off, for a file whose producer wrote wrong checksums. ### Encrypted workbooks -`open_workbook()`/`open_bytes()` take a `password` keyword to open a password-protected OOXML +`open_workbook()` takes a `password` keyword to open a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm): ```python @@ -491,7 +477,7 @@ An explicit `format="xlsx"`/`format="xlsb"` works for an encrypted file too, the `format` unset — both routes decrypt correctly given the right `password`. Writing an encrypted workbook is a second step: write the plaintext package with `write_workbook` -(or any of the `write_*` helpers), then wrap it with `encrypt_package`, which produces an +or `open_writer`, then wrap it with `encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container — AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: @@ -502,17 +488,21 @@ write_workbook("plain.xlsx", table, types) encrypt_package("plain.xlsx", "secret.xlsx", "hunter2") ``` -`package_path` is read twice (it is not disposed or removed), so it must already be a finished file. -Encryption parameters are fixed at Excel's own defaults — there are no options — and only XLSX/XLSB -packages can be encrypted, matching what `open_workbook`/`open_bytes` can decrypt. +The package must already be a finished file; it is read, not removed. Encryption parameters are +fixed at Excel's own defaults — there are no options — and only XLSX/XLSB packages can be encrypted, +matching what `open_workbook` can decrypt. -`encrypt_package_bytes()` does the same over bytes, so the plaintext package never has to be written -to disk: +Either side can stay in memory, so the plaintext package never has to be written to disk: the +package can be `bytes` or an `io.BytesIO`, and the destination any binary object with `write()`: ```python -from excelreader import encrypt_package_bytes, write_workbook_to_bytes +import io + +from excelreader import encrypt_package, write_workbook -secret = encrypt_package_bytes(write_workbook_to_bytes(table, types, format="xlsx"), "hunter2") +plain, secret = io.BytesIO(), io.BytesIO() +write_workbook(plain, table, types, format="xlsx") +encrypt_package(plain, secret, "hunter2") ``` ## Benchmarks @@ -528,32 +518,30 @@ fastexcel 0.21.0), pandas 3.0.5 (with openpyxl 3.1.5), xlsxwriter 3.2.9. | API | Median | What it produces | |---|---:|---| -| `to_record_batch_reader()` | 31.9 ms | Arrow record batches, streamed | +| `to_arrow_stream()`, read by pyarrow | 31.9 ms | Arrow record batches, streamed | | `to_arrow()` | 31.8 ms | the same parse, handed to pyarrow zero-copy | -| `parse_typed()` | 33.5 ms | typed columnar buffers, converted natively | -| `iter_parse_typed()` | 34.2 ms | the same typed buffers, in batches | +| `parse()` | 33.5 ms | typed columnar buffers, converted natively | +| `parse_batches()` | 34.2 ms | the same typed buffers, in batches | | `to_polars()` | 34.6 ms | typed columnar DataFrame, schema inferred | | `polars.read_excel()` | 125.6 ms | typed columnar DataFrame, types inferred | -| `read_all_columnar()` | 463.3 ms | raw columnar cells, no per-cell Python objects | | `rows()` | 1,034.6 ms | one `Cell` object per cell, streamed per row | -| `read_all()` | 1,631.0 ms | one `Cell` object per cell, all at once | The batched readers cost the same as their whole-sheet counterparts and lower the peak memory: -`to_record_batch_reader(batch_size=10000)` peaks at 11.4 MiB against 18.8 MiB for one whole-sheet +`to_arrow_stream(batch_size=10000)` peaks at 11.4 MiB against 18.8 MiB for one whole-sheet record batch on this file. Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison (~3.6x), and even that one is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules are not identical. The rows above it produce different things and are listed to show what each API -costs, not to rank them — `read_all()` is ~49x slower than `parse_typed()` because it materializes +costs, not to rank them — `rows()` is ~30x slower than `parse()` because it materializes 917,504 Python objects, which is the price of that shape, not a slow parser. CSV, from `65K_Records_Data.csv` (8.2 MB), same 14 typed columns: | API | Median | |---|---:| -| `parse_typed(parallelism=1)` | 23.6 ms | -| `parse_typed(parallelism=0)` | 12.0 ms | +| `parse(parallelism=1)` | 23.6 ms | +| `parse(parallelism=0)` | 12.0 ms | | `to_polars(parallelism=1)` | 25.0 ms | | `to_polars(parallelism=0)` | 10.8 ms | | `polars.read_csv(try_parse_dates=True)` | 7.3 ms | @@ -573,17 +561,17 @@ threads](#reading-a-large-sheet-on-several-threads). | `write_workbook()` → xls | 44.4 ms | 17.7 MB | | `write_workbook()` → xlsb | 71.5 ms | 5.1 MB | | `write_workbook()` → xlsx | 90.6 ms | 5.1 MB | -| `write_polars()` → xlsx | 487.4 ms | 5.1 MB | -| `write_pandas()` → xlsx | 493.8 ms | 5.1 MB | +| `write_workbook(polars_df)` → xlsx | 487.4 ms | 5.1 MB | +| `write_workbook(pandas_df)` → xlsx | 493.8 ms | 5.1 MB | | `polars.DataFrame.write_excel()` | 4,670.4 ms | 5.6 MB | | `pandas.DataFrame.to_excel()` | 7,192.9 ms | 5.5 MB | The two DataFrame comparisons are matched work — same DataFrame in, xlsx out both times: -`write_polars()` is ~9.6x faster than polars' own `write_excel()`, and `write_pandas()` ~14.6x -faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes +`write_workbook(polars_df)` is ~9.6x faster than polars' own `write_excel()`, and +`write_workbook(pandas_df)` ~14.6x faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes through Arrow and then a Python list before reaching the native columns, which is most of the gap between the ~490 ms rows and the 90 ms one. Handing `write_workbook()` buffers that are already -columnar — what `parse_typed()` returns — skips all of it. +columnar — what `parse()` returns — skips all of it. `write_workbook(xlsx)` at 90.6 ms lands within ~5 ms of the C++ binding's `xl::write_columns` on the same 14 columns (85.4 ms), which is the expected result: both are thin wrappers diff --git a/python/benchmarks/bench_polars_engine.py b/python/benchmarks/bench_polars_engine.py index 4e1f9b29..152a6c62 100644 --- a/python/benchmarks/bench_polars_engine.py +++ b/python/benchmarks/bench_polars_engine.py @@ -83,7 +83,7 @@ def _frame(rows: int) -> pl.DataFrame: def _generate(data_dir: Path, rows: int) -> dict[str, Path]: - from excelreader import encrypt_package, write_polars + from excelreader import encrypt_package, write_workbook data_dir.mkdir(parents=True, exist_ok=True) paths = { @@ -101,7 +101,7 @@ def _generate(data_dir: Path, rows: int) -> dict[str, Path]: if not paths["plain.xlsx"].exists(): df.write_excel(paths["plain.xlsx"], autofit=False) if not paths["plain.xlsb"].exists(): - write_polars(paths["plain.xlsb"], df) + write_workbook(paths["plain.xlsb"], df) for suffix in ("xlsx", "xlsb"): if not paths[f"encrypted.{suffix}"].exists(): encrypt_package(paths[f"plain.{suffix}"], paths[f"encrypted.{suffix}"], PASSWORD) diff --git a/python/benchmarks/bench_read.py b/python/benchmarks/bench_read.py index 9ca3f523..0412783d 100644 --- a/python/benchmarks/bench_read.py +++ b/python/benchmarks/bench_read.py @@ -4,12 +4,12 @@ `path` defaults to tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb, resolved relative to the repository root. Reports min and median over N iterations (default 10) for every available -read path, then a CSV comparison (`--csv`, default the fixture's .csv twin) of parse_typed() and +read path, then a CSV comparison (`--csv`, default the fixture's .csv twin) of parse() and to_polars() at parallelism 1 and 0 against polars.read_csv and pandas.read_csv. A variant that reads zero rows or zero cells is a harness bug, not a result — this script asserts against that instead of silently publishing a zero. -NOTE: read_all()/rows() build one Cell tuple per cell; polars.read_excel returns a typed columnar +NOTE: rows() builds one Cell tuple per cell; polars.read_excel returns a typed columnar DataFrame with type inference. That is not matched work (see STYLEGUIDE.md "Tests and Benchmarks") — this script labels the comparison, it does not pretend it is apples-to-apples. The parse_typed/parse_arrow variants are the closest thing to matched work against polars: they @@ -40,14 +40,6 @@ def _report(name: str, times: list[float], n: int) -> None: print(f"{name}: min={min(times):.4f}s median={statistics.median(times):.4f}s (n={n})") -def bench_read_all(path: Path) -> tuple[int, int]: - with excelreader.open_workbook(path) as workbook: - all_rows = workbook.sheets[0].read_all() - row_count = len(all_rows) - cell_count = sum(len(row) for row in all_rows) - return row_count, cell_count - - def bench_rows(path: Path) -> tuple[int, int]: with excelreader.open_workbook(path) as workbook: row_count = 0 @@ -58,12 +50,6 @@ def bench_rows(path: Path) -> tuple[int, int]: return row_count, cell_count -def bench_read_all_columnar(path: Path) -> tuple[int, int]: - with excelreader.open_workbook(path) as workbook: - sheet = workbook.sheets[0].read_all_columnar() - return len(sheet.row_offsets) - 1, len(sheet.columns) - - _T = excelreader.ColumnType _FIXTURE_SCHEMA = [ excelreader.ColumnSpec(type_, name=name) @@ -88,35 +74,39 @@ def bench_read_all_columnar(path: Path) -> tuple[int, int]: def bench_parse_typed(path: Path) -> tuple[int, int]: with excelreader.open_workbook(path) as workbook: - table = workbook.sheets[0].parse_typed(_FIXTURE_SCHEMA) + table = workbook.sheets[0].parse(_FIXTURE_SCHEMA) return table.row_count, table.row_count * len(table.columns) def bench_to_arrow(path: Path) -> tuple[int, int]: with excelreader.open_workbook(path) as workbook: - array = workbook.sheets[0].to_arrow(_FIXTURE_SCHEMA) - return len(array), len(array) * array.type.num_fields + batch = workbook.sheets[0].to_arrow(_FIXTURE_SCHEMA) + return batch.num_rows, batch.num_rows * batch.num_columns -def bench_iter_parse_typed(path: Path) -> tuple[int, int]: +def bench_parse_batches(path: Path) -> tuple[int, int]: """Batched counterpart to bench_parse_typed: one batch resident instead of the whole sheet.""" rows = 0 with excelreader.open_workbook(path) as workbook: - for table in workbook.sheets[0].iter_parse_typed(_FIXTURE_SCHEMA, batch_size=10000): + for table in workbook.sheets[0].parse_batches(_FIXTURE_SCHEMA, batch_size=10000): rows += table.row_count return rows, rows * len(_FIXTURE_SCHEMA) -def bench_record_batch_reader(path: Path) -> tuple[int, int]: +def _record_batch_reader(workbook): + import pyarrow + + return pyarrow.RecordBatchReader.from_stream(workbook.sheets[0].to_arrow_stream(_FIXTURE_SCHEMA, batch_size=10000)) + + +def bench_arrow_stream(path: Path) -> tuple[int, int]: rows = 0 with excelreader.open_workbook(path) as workbook: - for batch in workbook.sheets[0].to_record_batch_reader(_FIXTURE_SCHEMA, batch_size=10000): + for batch in _record_batch_reader(workbook): rows += batch.num_rows return rows, rows * len(_FIXTURE_SCHEMA) - - class _ProcessMemoryCountersEx(ctypes.Structure): _fields_ = [ ("cb", ctypes.c_uint32), @@ -149,7 +139,7 @@ def peak_leg_whole_sheet(path: Path) -> tuple[int, int]: """Whole-sheet Arrow read, sampled once while the full table is live. Returns (rows, peak).""" base = _private_bytes() with excelreader.open_workbook(path) as workbook: - batch = workbook.sheets[0].to_record_batch(_FIXTURE_SCHEMA) + batch = workbook.sheets[0].to_arrow(_FIXTURE_SCHEMA) peak = _private_bytes() - base rows = batch.num_rows return rows, peak @@ -161,7 +151,7 @@ def peak_leg_streamed(path: Path) -> tuple[int, int]: rows = 0 peak = 0 with excelreader.open_workbook(path) as workbook: - for batch in workbook.sheets[0].to_record_batch_reader(_FIXTURE_SCHEMA, batch_size=10000): + for batch in _record_batch_reader(workbook): rows += batch.num_rows peak = max(peak, _private_bytes() - base) del batch @@ -174,7 +164,7 @@ def peak_leg_streamed(path: Path) -> tuple[int, int]: def _measure_peaks(path: Path) -> dict[str, int]: """Runs each peak leg in a FRESH subprocess and collects its number. - In-process would read zero: the legs above allocate far more than these reads do (read_all alone + In-process would read zero: the legs above allocate far more than these reads do (rows() alone builds ~900k Cell tuples), a heap is not handed back to the OS on free, and so by the time we got here the growth we are trying to see would already be absorbed by slack. """ @@ -213,7 +203,7 @@ def bench_polars(path: Path) -> tuple[int, int]: def bench_csv_parse_typed(path: Path, parallelism: int) -> tuple[int, int]: with excelreader.open_workbook(path, format="csv") as workbook: - table = workbook.sheets[0].parse_typed(_FIXTURE_SCHEMA, parallelism=parallelism) + table = workbook.sheets[0].parse(_FIXTURE_SCHEMA, parallelism=parallelism) return table.row_count, table.row_count * len(table.columns) @@ -243,7 +233,7 @@ def _bench_csv(path: Path, n: int) -> None: print(f"csv: {path}") print() for parallelism in (1, 0): - print(f"excelreader.parse_typed(parallelism={parallelism}):") + print(f"excelreader.parse(parallelism={parallelism}):") _time_and_assert(f" csv parse_typed p={parallelism}", functools.partial(bench_csv_parse_typed, parallelism=parallelism), path, n) print() try: @@ -313,25 +303,17 @@ def main() -> int: print(f"file: {args.path}") print() - print("excelreader.read_all():") - _time_and_assert(" read_all", bench_read_all, args.path, args.n) - print() - print("excelreader.rows() (per-row iteration):") _time_and_assert(" rows", bench_rows, args.path, args.n) print() - print("excelreader.read_all_columnar() [no per-cell Cell/str objects — the fast path]:") - _time_and_assert(" read_all_columnar", bench_read_all_columnar, args.path, args.n) - print() - if args.path.resolve() == DEFAULT_PATH.resolve(): - print("parse_typed() [schema-driven typed columns, native-side conversion]:") + print("parse() [schema-driven typed columns, native-side conversion]:") _time_and_assert(" parse_typed", bench_parse_typed, args.path, args.n) print() - print("excelreader.iter_parse_typed() (batched):") - _time_and_assert(" iter_parse_typed", bench_iter_parse_typed, args.path, args.n) + print("excelreader.parse_batches() (batched):") + _time_and_assert(" parse_batches", bench_parse_batches, args.path, args.n) print() try: @@ -344,12 +326,12 @@ def main() -> int: _time_and_assert(" to_arrow", bench_to_arrow, args.path, args.n) print() - print("excelreader.to_record_batch_reader() (batched):") - _time_and_assert(" record_batch_reader", bench_record_batch_reader, args.path, args.n) + print("excelreader.to_arrow_stream() read by pyarrow (batched):") + _time_and_assert(" arrow_stream", bench_arrow_stream, args.path, args.n) print() if sys.platform == "win32": - print("peak memory [to_record_batch whole sheet vs to_record_batch_reader(batch_size=10000)]:") + print("peak memory [to_arrow whole sheet vs to_arrow_stream(batch_size=10000)]:") peaks = _measure_peaks(args.path) saved = peaks["whole_sheet"] - peaks["streamed"] print(f" streaming holds {saved} bytes ({saved / 1024 / 1024:.1f} MiB) less at the peak") diff --git a/python/benchmarks/bench_write.py b/python/benchmarks/bench_write.py index 00c14a90..939bf8cf 100644 --- a/python/benchmarks/bench_write.py +++ b/python/benchmarks/bench_write.py @@ -4,7 +4,7 @@ `path` is the READ source used to build the data this script writes back out — it defaults to the same 65K_Records_Data.xlsb fixture bench_read.py uses, resolved relative to the repository root. -The fixture is parsed ONCE via parse_typed()/to_pandas()/to_polars() before any timing starts; +The fixture is parsed ONCE via parse()/to_pandas()/to_polars() before any timing starts; every benchmark below writes that same in-memory table repeatedly, so what's measured is the write path alone, not read+write. Reports min and median over N iterations (default 10) for every available write path, into a fresh temp file per run. A write that produces an empty or missing @@ -76,16 +76,16 @@ def run() -> None: def benchmark_pandas(args: argparse.Namespace, tmp: Path): pd = find_spec("pandas") if pd is None: - print("pandas not installed — skipping write_pandas/to_excel comparison") + print("pandas not installed — skipping the pandas write/to_excel comparison") print() return with excelreader.open_workbook(args.path, format="xlsb") as workbook: pandas_df = workbook.sheets[0].to_pandas(_FIXTURE_SCHEMA) - print("write_pandas() [pandas.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") + print("write_workbook(pandas) [pandas.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") _time_and_assert( - " write_pandas", - lambda p: excelreader.write_pandas(p, pandas_df), + " write_workbook(pandas)", + lambda p: excelreader.write_workbook(p, pandas_df), tmp / "out_pandas.xlsx", args.n, ) @@ -93,7 +93,7 @@ def benchmark_pandas(args: argparse.Namespace, tmp: Path): openpyxl = find_spec("openpyxl") if openpyxl is None: - print("openpyxl not installed — skipping write_pandas/write_workbook comparison") + print("openpyxl not installed — skipping the pandas.DataFrame.to_excel() comparison") print() return print("pandas.DataFrame.to_excel() [same DataFrame, matched work — both write xlsx]:") @@ -108,17 +108,17 @@ def benchmark_pandas(args: argparse.Namespace, tmp: Path): def benchmark_polars(args: argparse.Namespace, tmp: Path): pl = find_spec("polars") if pl is None: - print("polars not installed — skipping write_polars/write_excel comparison") + print("polars not installed — skipping the polars write/write_excel comparison") print() return with excelreader.open_workbook(args.path, format="xlsb") as workbook: schema = workbook.sheets[0].infer_schema(sample_size=10) polars_df = workbook.sheets[0].to_polars(schema) - print("write_polars() [polars.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") + print("write_workbook(polars) [polars.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") _time_and_assert( - " write_polars", - lambda p: excelreader.write_polars(p, polars_df), + " write_workbook(polars)", + lambda p: excelreader.write_workbook(p, polars_df), tmp / "out_polars.xlsx", args.n, ) @@ -153,7 +153,7 @@ def main() -> int: print(f"source file: {args.path}") with excelreader.open_workbook(args.path, format="xlsb") as workbook: - table = workbook.sheets[0].parse_typed(_FIXTURE_SCHEMA) + table = workbook.sheets[0].parse(_FIXTURE_SCHEMA) print(f"parsed {table.row_count} rows x {len(table.columns)} columns once, before any timing") print() @@ -199,7 +199,7 @@ def main() -> int: try: import pyarrow # noqa: F401 except ImportError: - print("pyarrow not installed — skipping write_pandas/write_polars/pandas/polars comparisons") + print("pyarrow not installed — skipping the DataFrame writes and pandas/polars comparisons") return 0 benchmark_pandas(args, tmp) diff --git a/python/examples/read_workbook.py b/python/examples/read_workbook.py index d12fa407..46ffd727 100644 --- a/python/examples/read_workbook.py +++ b/python/examples/read_workbook.py @@ -14,7 +14,7 @@ def main() -> int: max_rows = int(sys.argv[2]) if len(sys.argv) > 2 else 10 with open_workbook(path) as workbook: - print(f"sheets={workbook.sheet_names} date1904={workbook.is_date1904}") + print(f"sheets={[sheet.name for sheet in workbook.sheets]} date1904={workbook.is_date1904}") for index, row in enumerate(workbook.sheets[0].rows()): if index >= max_rows: break diff --git a/python/hatch_build.py b/python/hatch_build.py index 9e4225a7..c74d27b3 100644 --- a/python/hatch_build.py +++ b/python/hatch_build.py @@ -1,35 +1,60 @@ """Forces a platform-specific wheel tag: the wheel bundles a native library per OS (see _lib/), so it isn't pure Python even though every .py file in it is. The package has no CPython C-extension (it's ctypes calling a NativeAOT shared library), so it's -ABI/interpreter-independent - only the platform component of the tag should be specific.""" +ABI/interpreter-independent - only the platform component of the tag should be specific. + +The platform component comes from the bundled library, not from the build machine: a +runner's own tag (macosx_26_0, manylinux_2_39) shuts out every older system the library +actually loads on.""" import platform +import re +import struct +from pathlib import Path from hatchling.builders.hooks.plugin.interface import BuildHookInterface from packaging.tags import platform_tags +LIB_DIR = Path(__file__).parent / "src" / "excelreader" / "_lib" +LC_BUILD_VERSION = 0x32 + + +def glibc_floor(library: bytes) -> str: + """The newest GLIBC_2.N symbol version the library needs, as manylinux's `2_N`.""" + minors = [int(minor) for minor in re.findall(rb"GLIBC_2\.(\d+)", library)] + return f"2_{max(minors, default=17)}" + + +def macos_floor(library: bytes) -> str: + """The minimum macOS a thin 64-bit Mach-O declares in LC_BUILD_VERSION, as `MAJOR_0`.""" + commands = struct.unpack_from("> 16}_0" + offset += size + raise ValueError("the native library has no LC_BUILD_VERSION load command") -def _platform_tag() -> str: - """Picks a platform tag PyPI actually accepts. - packaging.tags.platform_tags() lists manylinux/musllinux-compatible tags first - and the bare `linux_` tag last - but on a runner where it can't confirm - manylinux compliance (no _manylinux metadata), the bare tag is all that's offered, - and PyPI rejects raw `linux_*`/`freebsd_*`/etc. wheel uploads outright. Skip that - bare tag and fall back to manylinux2014 (glibc 2.17, released 2012), a floor every - current Linux distro satisfies - there's nothing to detect here since this wheel - never links against glibc at build time (it dlopen()s a NativeAOT .so at runtime). - """ +def platform_tag() -> str: machine = platform.machine().lower() - bare_linux = f"linux_{machine}" - for tag in platform_tags(): - if tag != bare_linux: - return tag - return f"manylinux2014_{machine}" + system = platform.system() + if system == "Linux": + library = (LIB_DIR / "ExcelReader.Native.so").read_bytes() + if b"libc.musl-" in library: + return next(tag for tag in platform_tags() if tag.startswith("musllinux_")) + return f"manylinux_{glibc_floor(library)}_{machine}" + if system == "Darwin": + library = (LIB_DIR / "ExcelReader.Native.dylib").read_bytes() + return f"macosx_{macos_floor(library)}_{machine}" + return next(iter(platform_tags())) class NativeLibraryBuildHook(BuildHookInterface): def initialize(self, version, build_data): build_data["pure_python"] = False build_data["infer_tag"] = True - build_data["tag"] = f"py3-none-{_platform_tag()}" + tag = next(iter(platform_tags())) if version == "editable" else platform_tag() + build_data["tag"] = f"py3-none-{tag}" diff --git a/python/pyproject.toml b/python/pyproject.toml index 9b06f7f6..51eefc98 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -7,7 +7,7 @@ name = "excelreader-native" version = "0.1.0" description = "Read XLSX, XLSB, XLS and CSV through the ExcelReader NativeAOT library" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "MIT" } authors = [{ name = "Gabriel Matte" }] classifiers = [ diff --git a/python/scripts/build_musl_wheel.sh b/python/scripts/build_musl_wheel.sh new file mode 100644 index 00000000..95448cb2 --- /dev/null +++ b/python/scripts/build_musl_wheel.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Builds the linux-musl native library and its musllinux wheel in Alpine containers, running the +# Python tests against it. Needs a Linux host with Docker; the wheel lands in dist/. +# +# Run from the repository root: sh python/scripts/build_musl_wheel.sh +set -eu + +docker run --rm -v "$PWD:/repo" -w /repo mcr.microsoft.com/dotnet/sdk:10.0-alpine \ + sh -c 'apk add --no-cache clang build-base zlib-dev python3 && python3 python/scripts/build_native.py' + +docker run --rm -v "$PWD:/repo" -w /repo python:3.12-alpine \ + sh -c 'pip install -e "python[dev]" build && pytest python/tests && python -m build --wheel python/ --outdir dist/ && python python/scripts/check_wheel_tag.py dist' diff --git a/python/scripts/build_native.py b/python/scripts/build_native.py index c85cf58d..319a3437 100644 --- a/python/scripts/build_native.py +++ b/python/scripts/build_native.py @@ -30,6 +30,8 @@ def default_rid() -> str: return f"win-{arch}" if system == "Darwin": return f"osx-{arch}" + if platform.libc_ver()[0] != "glibc": + return f"linux-musl-{arch}" return f"linux-{arch}" diff --git a/python/scripts/check_wheel_tag.py b/python/scripts/check_wheel_tag.py new file mode 100644 index 00000000..d5874a01 --- /dev/null +++ b/python/scripts/check_wheel_tag.py @@ -0,0 +1,29 @@ +"""Fails unless every wheel in the given directory installs on this machine. + +Run after building a wheel: python python/scripts/check_wheel_tag.py dist +""" + +import sys +from pathlib import Path + +from packaging.tags import sys_tags +from packaging.utils import parse_wheel_filename + + +def main() -> int: + supported = set(sys_tags()) + wheels = sorted(Path(sys.argv[1]).glob("*.whl")) + if not wheels: + print(f"error: no wheel in {sys.argv[1]}", file=sys.stderr) + return 1 + for wheel in wheels: + tags = parse_wheel_filename(wheel.name)[3] + if tags.isdisjoint(supported): + print(f"error: {wheel.name} does not install on this machine", file=sys.stderr) + return 1 + print(f"ok: {wheel.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/src/excelreader/__init__.py b/python/src/excelreader/__init__.py index 6b8457d1..ed6e12f4 100644 --- a/python/src/excelreader/__init__.py +++ b/python/src/excelreader/__init__.py @@ -1,12 +1,10 @@ """Python bindings for ExcelReader.""" -from excelreader.arrow_stream import ArrowStream -from excelreader.reader import Sheet, Sheets, Workbook, decode_cell, open_bytes, open_workbook -from excelreader.sources import Source, open_source, open_stream +from excelreader.reader import Sheet, Sheets, Workbook, open_workbook +from excelreader.sources import Source from excelreader.types import ( Cell, CellType, - ColumnarSheet, ColumnSpec, ColumnType, ExcelReaderError, @@ -18,27 +16,14 @@ TypedTable, WriteOptions, ) -from excelreader.writer import ( - encrypt_package, - encrypt_package_bytes, - write_arrow, - write_arrow_to_bytes, - write_pandas, - write_pandas_to_bytes, - write_polars, - write_polars_to_bytes, - write_workbook, - write_workbook_to_bytes, -) -from excelreader.writer_handle import SheetWriter, open_writer, open_writer_to_memory +from excelreader.writer import encrypt_package, write_workbook +from excelreader.writer_handle import SheetWriter, open_writer __all__ = [ - "ArrowStream", "Cell", "CellType", "ColumnSpec", "ColumnType", - "ColumnarSheet", "ExcelReaderError", "OpenOptions", "PasswordIncorrectError", @@ -52,21 +37,8 @@ "TypedTable", "Workbook", "WriteOptions", - "decode_cell", "encrypt_package", - "encrypt_package_bytes", - "open_bytes", - "open_source", - "open_stream", "open_workbook", "open_writer", - "open_writer_to_memory", - "write_arrow", - "write_arrow_to_bytes", - "write_pandas", - "write_pandas_to_bytes", - "write_polars", - "write_polars_to_bytes", "write_workbook", - "write_workbook_to_bytes", ] diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index cf8f5b25..e39b2e50 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -60,22 +60,6 @@ XL_T_TIMESTAMP = 6 -class NativeRowCell(ctypes.Structure): - _fields_ = [ - ("column", ctypes.c_int32), - ("type", ctypes.c_int32), - ("value_len", ctypes.c_int32), - ("value", ctypes.POINTER(ctypes.c_uint8)), - ] - - -class NativeRow(ctypes.Structure): - _fields_ = [ - ("cell_count", ctypes.c_int32), - ("cells", ctypes.POINTER(NativeRowCell)), - ] - - class NativeColumnSpec(ctypes.Structure): """Mirrors xl_column_spec. `names`/`name_lens`/`name_count` may be left NULL/NULL/0 to resolve by `index` instead. Build one with `column_spec_by_name`/`column_spec_by_names`, never directly — @@ -225,15 +209,6 @@ class ArrowArrayStream(ctypes.Structure): ] -class NativeRows(ctypes.Structure): - """Mirrors xl_rows_decoded.""" - - _fields_ = [ - ("row_count", ctypes.c_int32), - ("rows", ctypes.POINTER(NativeRow)), - ] - - class NativeBuffer(ctypes.Structure): """Mirrors xl_buffer.""" @@ -332,13 +307,14 @@ def to_native_open_options(options: OpenOptions) -> NativeOpenOptions: raw = default_open_options() raw.csv_sniff_dialect = _opt_state(options.csv_sniff_dialect) raw.csv_detect_bom = _opt_state(options.csv_detect_bom) - raw.csv_intern_strings = _opt_state(options.csv_intern_strings) raw.prefetch_decompression = _opt_state(options.prefetch_decompression) raw.intern_strings = _opt_state(options.intern_strings) raw.verify_zip_checksums = _opt_state(options.verify_zip_checksums) raw.csv_delimiter = _opt_number(options.csv_delimiter) raw.csv_quote = _opt_number(options.csv_quote) - raw.csv_max_cell_bytes = _opt_number(options.csv_max_cell_bytes) + # The ABI's csv_* copies win over these for a CSV even when unset, so send the same value to both. + raw.csv_max_cell_bytes = _opt_number(options.max_cell_bytes) + raw.csv_intern_strings = raw.intern_strings raw.max_total_decompressed_bytes = _opt_number(options.max_total_decompressed_bytes) raw.max_cell_bytes = _opt_number(options.max_cell_bytes) raw.max_shared_string_bytes = _opt_number(options.max_shared_string_bytes) @@ -495,12 +471,6 @@ def _bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.xl_rows_close.restype = c_int lib.xl_rows_next.argtypes = [p_void, p_bytes, c_int, p_int] lib.xl_rows_next.restype = c_int - lib.xl_rows_read_all_blob.argtypes = [p_void, p_bytes, c_int, p_int] - lib.xl_rows_read_all_blob.restype = c_int - lib.xl_rows_read_all_decoded.argtypes = [p_void, ctypes.POINTER(NativeRows)] - lib.xl_rows_read_all_decoded.restype = c_int - lib.xl_free_rows.argtypes = [ctypes.POINTER(NativeRows)] - lib.xl_free_rows.restype = None lib.xl_parse_typed.argtypes = [p_void, c_int, p_specs, c_int, c_int, c_int, ctypes.POINTER(NativeTable)] lib.xl_parse_typed.restype = c_int lib.xl_typed_reader_open.argtypes = [p_void, c_int, p_specs, c_int, c_int, ctypes.c_int64, pp_void] diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index ca87f8ef..bb07f645 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -3,23 +3,26 @@ from __future__ import annotations import ctypes +import io import operator +import os import struct import threading from array import array from collections.abc import Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, SupportsIndex, overload +from typing import IO, TYPE_CHECKING, SupportsIndex, overload if TYPE_CHECKING: from typing_extensions import Self + from excelreader.sources import Source + from excelreader import _native from excelreader.arrow_stream import ArrowStream from excelreader.types import ( Cell, CellType, - ColumnarSheet, ColumnSpec, ColumnType, ExcelReaderError, @@ -40,7 +43,6 @@ _CELL_HEADER = struct.Struct(" str: @@ -105,24 +107,21 @@ def _require_handle(self) -> ctypes.c_void_p: raise ExcelReaderError("workbook is closed") return self._handle - @property - def sheet_count(self) -> int: + def _sheet_count(self) -> int: count = ctypes.c_int32() _check(self._lib.xl_sheet_count(self._require_handle(), ctypes.byref(count))) return count.value - def sheet_name_at(self, index: int) -> str: + def _sheet_name_at(self, index: int) -> str: raw, length = _fill_buffer(self._lib.xl_sheet_name_at, self._require_handle(), index, initial=256) return raw[:length].decode("utf-8") - @property - def sheet_names(self) -> list[str]: - """Every sheet's name, in order.""" - return [self.sheet_name_at(index) for index in range(self.sheet_count)] - @property def sheets(self) -> Sheets: - """The workbook's sheets: `len()`, iteration, and lookup by index, slice or name.""" + """The workbook's sheets: `len()`, iteration, and lookup by index, slice or name. + + `[sheet.name for sheet in workbook.sheets]` lists every sheet's name. + """ return Sheets(self) @property @@ -132,7 +131,7 @@ def is_date1904(self) -> bool: return flag.value != 0 def close(self) -> None: - """Closes the workbook. A generator that has already started, and any record-batch reader already opened, keep working.""" + """Closes the workbook. A generator that has already started, and any Arrow stream already opened, keep working.""" with self._close_lock: handle, self._handle = self._handle, None if handle is None: @@ -159,7 +158,7 @@ def __init__(self, workbook: Workbook) -> None: self._workbook = workbook def __len__(self) -> int: - return self._workbook.sheet_count + return self._workbook._sheet_count() @overload def __getitem__(self, key: SupportsIndex | str) -> Sheet: ... @@ -215,7 +214,7 @@ def index(self) -> int: @property def name(self) -> str: - return self._workbook.sheet_name_at(self._index) + return self._workbook._sheet_name_at(self._index) @property def visibility(self) -> SheetVisibility: @@ -235,6 +234,7 @@ def _close_rows(self, cursor: ctypes.c_void_p) -> None: def rows(self) -> Iterator[list[Cell]]: """This sheet's rows, from the first. Each call is an independent pass over the sheet.""" + date1904 = self._workbook.is_date1904 cursor = self._open_rows() try: written = ctypes.c_int32() @@ -249,45 +249,15 @@ def rows(self) -> Iterator[list[Cell]]: buffer = ctypes.create_string_buffer(capacity) continue _check(status) - yield _decode_row(buffer.raw, written.value) + yield _decode_row(buffer.raw, written.value, date1904) finally: self._close_rows(cursor) - def read_all(self) -> list[list[Cell]]: - """Materializes every row of this sheet in one native call. - - Builds one `Cell`/`str` object per cell — for a large sheet, `read_all_columnar()` is - substantially faster because it never does that. - """ - cursor = self._open_rows() - try: - rows = _native.NativeRows() - _check(self._lib.xl_rows_read_all_decoded(cursor, ctypes.byref(rows))) - try: - return [_decode_native_row(rows.rows[index]) for index in range(rows.row_count)] - finally: - self._lib.xl_free_rows(ctypes.byref(rows)) - finally: - self._close_rows(cursor) - - def read_all_columnar(self) -> ColumnarSheet: - """Materializes every row of this sheet as parallel flat arrays. - - The fast path for large sheets: unlike `read_all()`, this never constructs a `Cell`/`str` - per cell. Use `decode_cell()` to materialize one cell on demand. - """ - cursor = self._open_rows() - try: - raw, written = _fill_buffer(self._lib.xl_rows_read_all_blob, cursor, initial=_INITIAL_ALL_ROWS_BUFFER) - finally: - self._close_rows(cursor) - return _decode_columnar(raw, written) - - def parse_typed(self, schema: Sequence[ColumnSpec], header_row: int = 1, parallelism: int = 1) -> TypedTable: + def parse(self, schema: Sequence[ColumnSpec], *, header_row: int = 1, parallelism: int = 1) -> TypedTable: """Reads this whole sheet into typed columns, converting values on the native side. - By far the fastest path in this library: unlike every other read method, no cell is ever - formatted to text, so a large sheet costs a fraction of `read_all_columnar()`. + By far the fastest path in this library: unlike `rows()`, no cell is ever formatted to text + or turned into a Python object. `header_row` is a 1-based row number whose values name the columns (that row is skipped and never yielded as data); 0 means the sheet has no header, in which case every spec must @@ -307,17 +277,17 @@ def parse_typed(self, schema: Sequence[ColumnSpec], header_row: int = 1, paralle finally: self._lib.xl_free_table(ctypes.byref(table)) - def iter_parse_typed( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 + def parse_batches( + self, schema: Sequence[ColumnSpec], *, header_row: int = 1, batch_size: int = 10000 ) -> Iterator[TypedTable]: - """`parse_typed()` a batch at a time, yielding one `TypedTable` per batch. + """`parse()` a batch at a time, yielding one `TypedTable` per batch. - `batch_size` is rows per batch; 0 means one unbounded batch, exactly what `parse_typed()` + `batch_size` is rows per batch; 0 means one unbounded batch, exactly what `parse()` does. A generator that has already started owns its position in the sheet and keeps working after the workbook is closed. Being a generator, it opens nothing until the first iteration, so a bad `batch_size` is not - reported until then; `to_record_batch_reader()` raises immediately instead. + reported until then; `to_arrow_stream()` raises immediately instead. """ handle = self._workbook._require_handle() specs = _build_specs(schema) @@ -341,58 +311,24 @@ def iter_parse_typed( finally: self._lib.xl_typed_reader_close(reader) - def to_arrow(self, schema: Sequence[ColumnSpec], header_row: int = 1, parallelism: int = 1) -> object: - """The same read as `parse_typed()`, handed to pyarrow as one `StructArray`, zero-copy. + def to_arrow(self, schema: Sequence[ColumnSpec], *, header_row: int = 1, parallelism: int = 1) -> object: + """The same read as `parse()`, handed to pyarrow as one column-named `RecordBatch`, zero-copy. - Requires pyarrow. The returned array owns the native buffers through the Arrow C Data - Interface's release callback, so it stays valid after this workbook is closed. Wrap it with - `pyarrow.RecordBatch.from_struct_array()` for a column-named batch. `parallelism` works as in - `parse_typed()`. + Requires pyarrow. The batch owns the native buffers through the Arrow C Data Interface's + release callback, so it stays valid after this workbook is closed. `parallelism` works as in + `parse()`. To stream instead, see `to_arrow_stream()`. """ - try: - import pyarrow - except ImportError: - raise ImportError( - "to_arrow() requires pyarrow — install it with `pip install pyarrow`, or use " - "parse_typed(), which returns the same data with no third-party dependency." - ) from None - + pyarrow = _import_pyarrow("to_arrow()", "parse()") handle = self._workbook._require_handle() specs = _build_specs(schema) array = _native.ArrowArray() arrow_schema = _native.ArrowSchema() _check(self._lib.xl_parse_arrow(handle, self._index, specs, len(specs), header_row, parallelism, ctypes.byref(array), ctypes.byref(arrow_schema))) - return pyarrow.Array._import_from_c(ctypes.addressof(array), ctypes.addressof(arrow_schema)) - - def to_record_batch(self, schema: Sequence[ColumnSpec], header_row: int = 1, parallelism: int = 1) -> object: - """Same read as `to_arrow()`, wrapped as a column-named `pyarrow.RecordBatch`. - - Requires pyarrow — see `to_arrow()`. - """ - array = self.to_arrow(schema, header_row=header_row, parallelism=parallelism) - import pyarrow - - return pyarrow.RecordBatch.from_struct_array(array) - - def to_record_batch_reader( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 - ) -> object: - """The same read as `to_arrow()`, as a streaming `pyarrow.RecordBatchReader`. - - Peak memory is one batch rather than one sheet. Requires pyarrow. - - pyarrow owns the stream once it is imported — exhausting or dropping the reader is what - closes the underlying read. - """ - try: - import pyarrow - except ImportError: - raise ImportError( - "to_record_batch_reader() requires pyarrow — install it with `pip install " - "pyarrow`, or use iter_parse_typed(), which streams the same data with no " - "third-party dependency." - ) from None + return pyarrow.RecordBatch._import_from_c(ctypes.addressof(array), ctypes.addressof(arrow_schema)) + def _open_arrow_stream( + self, schema: Sequence[ColumnSpec], header_row: int, batch_size: int + ) -> _native.ArrowArrayStream: handle = self._workbook._require_handle() specs = _build_specs(schema) stream = _native.ArrowArrayStream() @@ -401,85 +337,47 @@ def to_record_batch_reader( handle, self._index, specs, len(specs), header_row, batch_size, ctypes.byref(stream) ) ) - return pyarrow.RecordBatchReader._import_from_c(ctypes.addressof(stream)) + return stream def to_arrow_stream( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 + self, schema: Sequence[ColumnSpec], *, header_row: int = 1, batch_size: int = 10000 ) -> ArrowStream: - """The same read as `to_record_batch_reader()`, as a single-use Arrow PyCapsule stream. - - Needs no third-party package to create; pass it to any consumer of `__arrow_c_stream__` - (`polars.DataFrame(stream)`, `pyarrow.RecordBatchReader.from_stream(stream)`, ...). - """ - handle = self._workbook._require_handle() - specs = _build_specs(schema) - stream = _native.ArrowArrayStream() - _check( - self._lib.xl_parse_arrow_stream( - handle, self._index, specs, len(specs), header_row, batch_size, ctypes.byref(stream) - ) - ) - return ArrowStream(stream) - - def iter_pandas( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 - ) -> Iterator[object]: - """One `pandas.DataFrame` per batch, streamed. Requires pyarrow and pandas. + """The same read as `to_arrow()`, streamed `batch_size` rows at a time as a single-use Arrow + PyCapsule stream. Peak memory is one batch rather than one sheet. - A generator, so a missing dependency is reported at the first iteration, not at the call. + Needs no third-party package to create; pass it to any consumer of `__arrow_c_stream__`: + `polars.DataFrame(stream)`, or `pyarrow.RecordBatchReader.from_stream(stream)` for a + `RecordBatchReader`. The stream stays valid after this workbook is closed. """ - try: - import pyarrow # noqa: F401 - except ImportError: - raise ImportError( - "iter_pandas() requires pyarrow — install it with `pip install pyarrow`, or use " - "iter_parse_typed(), which streams the same data with no third-party dependency." - ) from None - - reader = self.to_record_batch_reader(schema, header_row=header_row, batch_size=batch_size) - for batch in reader: - yield batch.to_pandas(self_destruct=True, split_blocks=True) - - def iter_polars( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 - ) -> Iterator[object]: - """One `polars.DataFrame` per batch, streamed. Requires pyarrow and polars. - - A generator, so a missing dependency is reported at the first iteration, not at the call. - """ - try: - import polars - except ImportError: - raise ImportError( - "iter_polars() requires polars — install it with `pip install polars`, or use " - "to_record_batch_reader(), which streams the same data with no polars dependency." - ) from None - - reader = self.to_record_batch_reader(schema, header_row=header_row, batch_size=batch_size) - for batch in reader: - yield polars.from_arrow(batch, rechunk=False) + return ArrowStream(self._open_arrow_stream(schema, header_row, batch_size)) def to_pandas( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 + self, schema: Sequence[ColumnSpec], *, header_row: int = 1, batch_size: int = 10000, parallelism: int = 1 ) -> object: - """Same read as `to_arrow()`, materialized as a `pandas.DataFrame`. + """Same read as `to_arrow()`, materialized as a `pandas.DataFrame`. Requires pyarrow and pandas. - `read_all()` concatenates every batch, so this does NOT bound peak memory the way - `iter_pandas()` does. What it buys is the conversion: `self_destruct` frees each Arrow - chunk as pandas takes it, so the sheet is never held twice. Requires pyarrow and pandas. + Streams `batch_size` rows at a time, then concatenates them, so peak memory is not bounded + by the batch size; `self_destruct` frees each Arrow chunk as pandas takes it, so the sheet + is never held twice. With `parallelism` other than 1 the sheet is parsed whole, as in + `parse()`, instead of streamed. """ - reader = self.to_record_batch_reader(schema, header_row=header_row, batch_size=batch_size) - return reader.read_all().to_pandas(self_destruct=True, split_blocks=True) + pyarrow = _import_pyarrow("to_pandas()", "parse()") + if parallelism != 1: + data = self.to_arrow(schema, header_row=header_row, parallelism=parallelism) + else: + stream = self._open_arrow_stream(schema, header_row, batch_size) + data = pyarrow.RecordBatchReader._import_from_c(ctypes.addressof(stream)).read_all() + return data.to_pandas(self_destruct=True, split_blocks=True) def to_polars( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000, parallelism: int = 1 + self, schema: Sequence[ColumnSpec], *, header_row: int = 1, batch_size: int = 10000, parallelism: int = 1 ) -> object: """Same read as `to_arrow()`, materialized as a `polars.DataFrame`, zero-copy. Streams internally through `to_arrow_stream()` — polars consumes the stream a batch at a time, so unlike `to_pandas()` the whole sheet is never resident as Arrow buffers. Requires polars only, not pyarrow. With `parallelism` other than 1 the sheet is parsed whole, as in - `parse_typed()`, instead of streamed, and that path does require pyarrow. + `parse()`, instead of streamed, and that path does require pyarrow. """ try: import polars @@ -490,12 +388,11 @@ def to_polars( ) from None if parallelism != 1: - return polars.from_arrow(self.to_record_batch(schema, header_row=header_row, parallelism=parallelism)) - stream = self.to_arrow_stream(schema, header_row=header_row, batch_size=batch_size) - return polars.DataFrame(stream) + return polars.from_arrow(self.to_arrow(schema, header_row=header_row, parallelism=parallelism)) + return polars.DataFrame(self.to_arrow_stream(schema, header_row=header_row, batch_size=batch_size)) - def infer_schema(self, header_row: int = 1, sample_size: int = 100, parse_text: bool = False) -> list[ColumnSpec]: - """Guesses a `parse_typed()`/`to_arrow()` schema by sampling this sheet's cells. + def infer_schema(self, *, header_row: int = 1, sample_size: int = 100, parse_text: bool = False) -> list[ColumnSpec]: + """Guesses a `parse()`/`to_arrow()` schema by sampling this sheet's cells. Reads `header_row` for column names (0 means no header — every returned spec resolves by `index` instead) and up to `sample_size` rows after it, guessing each column's type from @@ -509,7 +406,7 @@ def infer_schema(self, header_row: int = 1, sample_size: int = 100, parse_text: numbers and non-ISO dates stay `ColumnType.STRING`. This is a guess over a sample, not a guarantee — a column that looks like `ColumnType.I64` in - the sample can still hold a fractional value further down the sheet, which `parse_typed()` + the sample can still hold a fractional value further down the sheet, which `parse()` would then reject unless the spec is `nullable`. """ handle = self._workbook._require_handle() @@ -522,7 +419,7 @@ def infer_schema(self, header_row: int = 1, sample_size: int = 100, parse_text: self._lib.xl_free_schema(ctypes.byref(schema)) -def _decode_row(blob: bytes, length: int) -> list[Cell]: +def _decode_row(blob: bytes, length: int, date1904: bool) -> list[Cell]: count = struct.unpack_from(" list[Cell]: offset += _CELL_HEADER.size value = blob[offset : offset + value_length].decode("utf-8") offset += value_length - cells.append(Cell(column=column, type=CellType(cell_type), value=value)) + cells.append(Cell(column=column, type=CellType(cell_type), value=value, date1904=date1904)) if offset != length: raise ExcelReaderError(f"row blob is malformed: consumed {offset} of {length} bytes") return cells -def _decode_native_row(row: _native.NativeRow) -> list[Cell]: - cells: list[Cell] = [] - for index in range(row.cell_count): - cell = row.cells[index] - value = ctypes.string_at(cell.value, cell.value_len).decode("utf-8") - cells.append(Cell(column=cell.column, type=CellType(cell.type), value=value)) - return cells - - -def _decode_columnar(blob: bytes, length: int) -> ColumnarSheet: - # expensive. (ponytail: if this loop measurably dominates for very wide/tall sheets, revisit with - row_offsets = array("i", [0]) - columns = array("i") - types = array("i") - value_offsets = array("i", [0]) - values = bytearray() - - row_count = struct.unpack_from(" object: + try: + import pyarrow + except ImportError: + raise ImportError( + f"{method} requires pyarrow — install it with `pip install pyarrow`, or use " + f"{alternative}, which returns the same data with no third-party dependency." + ) from None + return pyarrow def _decode_inferred_schema(schema: _native.NativeInferredSchema) -> list[ColumnSpec]: @@ -671,19 +525,6 @@ def _decode_validity(column: _native.NativeColumn, row_count: int) -> bytes | No return ctypes.string_at(column.validity, (row_count + 7) // 8) -def _to_columnar_array(values: array) -> object: - if _numpy is None: - return values - return _numpy.frombuffer(values, dtype=_numpy.int32) - - -def decode_cell(sheet: ColumnarSheet, index: int) -> Cell: - """Materializes the `Cell` at flat cell index `index` in `sheet`, decoding only that one value.""" - start, end = int(sheet.value_offsets[index]), int(sheet.value_offsets[index + 1]) - value = bytes(sheet.values[start:end]).decode("utf-8") - return Cell(column=int(sheet.columns[index]), type=CellType(int(sheet.types[index])), value=value) - - def _raw_options( options: OpenOptions | None, password: str | None ) -> tuple[object, _native.NativeOpenOptions | None, ctypes.Array[ctypes.c_char] | None]: @@ -711,43 +552,68 @@ def _raw_options( def open_workbook( - path: str | Path, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None + source: str | os.PathLike[str] | bytes | bytearray | memoryview | IO[bytes] | Source, + format: str | None = None, + *, + options: OpenOptions | None = None, + password: str | None = None, ) -> Workbook: - """Opens a workbook from disk. `format` is one of auto/xls/xlsx/xlsb/csv; None infers it. - - `options` overrides reader limits and CSV dialect settings; see `OpenOptions`. `password` unlocks - an encrypted OOXML workbook (.xlsx/.xlsb); omitting it for one raises `PasswordRequiredError`, and - a wrong one raises `PasswordIncorrectError`. An explicit xlsx/xlsb `format` works for an encrypted - file too, the same as leaving `format` unset — both routes decrypt correctly given the right - `password`. + """Opens a workbook from a path, bytes in memory, a `Source`, or a readable stream. + + `source` picks how the workbook is read: + + - a `str` or path-like: the file on disk. + - `bytes`, `bytearray`, `memoryview` or `io.BytesIO`: a copy the native side takes immediately. + A `BytesIO` is read whole, whatever its position. + - a `Source` (`size` and `read_at`): fetched in blocks on demand, and every sheet can be read on + its own thread. The library keeps a reference to it until the workbook and everything read + from it are closed. `read_at` runs with the GIL held, so a local file is better opened by path. + - any other object with `readinto()` or `read()`, such as an open file, a pipe or + `sys.stdin.buffer`: read as a stream that cannot seek. A CSV is read as it arrives, and can be + read again from the start (`infer_schema()`, then a read) until 16 MiB of it were read; an + XLSX, XLSB or XLS is read whole before the workbook opens. The library never closes it. + + `format` is one of auto/xls/xlsx/xlsb/csv; None infers it, from the extension for a path and from + the content otherwise. The content tells XLS, XLSX and XLSB apart, and a stream that is none of + them reads as CSV; a CSV in memory or behind a `Source` needs `format="csv"`. `options` overrides reader limits and CSV dialect settings; see + `OpenOptions`. `password` unlocks an encrypted OOXML workbook (.xlsx/.xlsb); omitting it for one + raises `PasswordRequiredError`, and a wrong one raises `PasswordIncorrectError`. """ - resolved = Path(path) - encoded = str(resolved).encode("utf-8") + if isinstance(source, (str, os.PathLike)): + return _open_path(Path(source), format, options, password) + if isinstance(source, io.BytesIO): + source = source.getvalue() + if isinstance(source, (bytes, bytearray, memoryview)): + return _open_memory(source, format, options, password) + + from excelreader import sources + + if hasattr(source, "read_at"): + return sources._open_source(source, format, options, password) + if hasattr(source, "readinto") or hasattr(source, "read"): + return sources._open_stream(source, format, options, password) + raise TypeError( + "open_workbook() needs a path, bytes, io.BytesIO, a Source, or an object with readinto() or read();" + f" got {type(source).__name__}" + ) + + +def _open_path(path: Path, format: str | None, options: OpenOptions | None, password: str | None) -> Workbook: + encoded = str(path).encode("utf-8") handle = ctypes.c_void_p() lib = _native.load_library() raw_options, _raw, _password_buffer = _raw_options(options, password) - _check( - lib.xl_open_file( - encoded, len(encoded), _resolve_format(format, resolved), raw_options, ctypes.byref(handle) - ) - ) + _check(lib.xl_open_file(encoded, len(encoded), _resolve_format(format, path), raw_options, ctypes.byref(handle))) return Workbook(handle) -def open_bytes( - data: bytes, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +def _open_memory( + data: bytes | bytearray | memoryview, format: str | None, options: OpenOptions | None, password: str | None ) -> Workbook: - """Opens a workbook from an in-memory buffer. The native side copies `data` immediately. - - `options` overrides reader limits and CSV dialect settings; see `OpenOptions`. `password` unlocks - an encrypted OOXML workbook — see `open_workbook()` for the full contract. - """ + if not isinstance(data, bytes): + data = bytes(data) handle = ctypes.c_void_p() lib = _native.load_library() raw_options, _raw, _password_buffer = _raw_options(options, password) - _check( - lib.xl_open_memory( - data, len(data), _resolve_format(format, None), raw_options, ctypes.byref(handle) - ) - ) + _check(lib.xl_open_memory(data, len(data), _resolve_format(format, None), raw_options, ctypes.byref(handle))) return Workbook(handle) diff --git a/python/src/excelreader/sources.py b/python/src/excelreader/sources.py index faf39835..701be0f0 100644 --- a/python/src/excelreader/sources.py +++ b/python/src/excelreader/sources.py @@ -1,4 +1,4 @@ -"""Open a workbook from bytes the caller serves: a positional `Source`, or a readable stream.""" +"""Reading a workbook through callbacks: a positional `Source`, or a readable stream.""" from __future__ import annotations @@ -13,7 +13,7 @@ class Source(Protocol): - """Random-access bytes for `open_source()`. `read_at` is called from several threads at once.""" + """Random-access bytes for `open_workbook()`. `read_at` is called from several threads at once.""" size: int @@ -60,7 +60,7 @@ def _read(key: int, buffer: int, length: int) -> int: return len(data) count = readinto(_view(buffer, length)) if count is None: - raise OSError("the stream has no data ready; open_stream() needs a blocking stream") + raise OSError("the stream has no data ready; open_workbook() needs a blocking stream") count = operator.index(count) if count > length: raise OSError("the stream returned more bytes than requested") @@ -74,14 +74,9 @@ def _release(key: int) -> None: _live.pop(key, None) -def open_source( - source: Source, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +def _open_source( + source: Source, format: str | None, options: OpenOptions | None, password: str | None ) -> Workbook: - """Opens a workbook over bytes `source` serves; every sheet can be read on its own thread. - - The library keeps a reference to `source` until the workbook and everything read from it are - closed. `read_at` runs with the GIL held, so a local file is better opened with `open_workbook()`. - """ size = operator.index(source.size) if size < 0: raise ValueError("source.size must not be negative") @@ -96,18 +91,9 @@ def open_source( return Workbook(handle) -def open_stream( - stream: object, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +def _open_stream( + stream: object, format: str | None, options: OpenOptions | None, password: str | None ) -> Workbook: - """Opens a workbook over a stream that cannot seek, such as `sys.stdin.buffer` or a socket. - - A CSV (`format="csv"`, or no format and neither a ZIP nor an OLE2 signature) is read as it - arrives, and can be read again from the start - (`infer_schema()`, then a read) until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole - before the workbook opens. `stream` needs `readinto` or `read`; the library never closes it. - """ - if not (hasattr(stream, "readinto") or hasattr(stream, "read")): - raise TypeError("open_stream() needs an object with readinto() or read()") lib = _native.load_library() resolved = _resolve_format(format, None) raw_options, _raw, _password_buffer = _raw_options(options, password) diff --git a/python/src/excelreader/types.py b/python/src/excelreader/types.py index 5a143485..e486442f 100644 --- a/python/src/excelreader/types.py +++ b/python/src/excelreader/types.py @@ -3,8 +3,9 @@ from __future__ import annotations import datetime +from dataclasses import KW_ONLY, dataclass from enum import IntEnum -from typing import Any, NamedTuple +from typing import Any from excelreader import _native @@ -35,17 +36,22 @@ class SheetVisibility(IntEnum): _MAX_SERIAL = 2_958_466 -class Cell(NamedTuple): - """One cell. `value` is the raw text as stored, so DATE cells are Excel serial numbers.""" +@dataclass(slots=True, kw_only=True) +class Cell: + """One cell. `value` is the raw text as stored, so DATE cells are Excel serial numbers; + `date1904` is the date system of the workbook it came from. + + Unlike the other value types it is not frozen: `rows()` builds one per cell, and a frozen + dataclass's `__init__` costs twice as much.""" column: int type: CellType value: str + date1904: bool = False - def as_date(self, date1904: bool = False) -> datetime.date | None: + def as_date(self) -> datetime.date | None: """Converts a DATE cell's serial value to a date. Returns None for any other cell type. - `date1904` should come from `Workbook.is_date1904` for the workbook this cell came from. Raises `ValueError` for a serial outside what Excel can show as a date. """ if self.type is not CellType.DATE: @@ -54,7 +60,7 @@ def as_date(self, date1904: bool = False) -> datetime.date | None: if not 0 <= serial < _MAX_SERIAL: raise ValueError(f"{self.value} is not an Excel date serial") days = int(serial) - if date1904: + if self.date1904: return datetime.date(1904, 1, 1) + datetime.timedelta(days=days) # Excel counts a 29 February 1900 that never existed, so serials before it sit one day later # against this epoch than those after. Serial 60, that day itself, reads as 28 February. @@ -63,33 +69,8 @@ def as_date(self, date1904: bool = False) -> datetime.date | None: return datetime.date(1899, 12, 30) + datetime.timedelta(days=days) -class ColumnarSheet(NamedTuple): - """Every cell of a sheet as parallel flat arrays, produced by `Sheet.read_all_columnar()`. - - The fast path for large sheets: unlike `Sheet.read_all()`, decoding never constructs one - `Cell`/`str` object per cell — only `int` values fill the index arrays below, and `values` holds - every cell's raw UTF-8 bytes concatenated. Use `decode_cell()` to materialize a single `Cell` on - demand, or slice `values` directly for bulk work. - - Each array is `array.array('i')` normally, or a NumPy `int32` `ndarray` when NumPy is installed - (see `reader.py`) — both support the same `len()`/indexing/slicing used here. - - - `row_offsets[i] : row_offsets[i + 1]` are the cell indices belonging to row `i`, indexing into - `columns`/`types`/`value_offsets`. Length `row_count + 1`. - - `columns[j]` / `types[j]` are cell `j`'s column index and `CellType` value. Length `cell_count`. - - `value_offsets[j] : value_offsets[j + 1]` is cell `j`'s byte slice into `values`. Length - `cell_count + 1`. - """ - - row_offsets: Any - columns: Any - types: Any - value_offsets: Any - values: bytes - - class ColumnType(IntEnum): - """A column's requested type in a `Sheet.parse_typed()`/`to_arrow()` schema. + """A column's requested type in a `Sheet.parse()`/`to_arrow()` schema. Mirrors XL_T_* in src/ExcelReader.Native/include/excelreader.h — the values are ABI, do not renumber them. The buffer layout each one produces is documented on `TypedTable.columns`. @@ -104,7 +85,8 @@ class ColumnType(IntEnum): TIMESTAMP = 6 -class ColumnSpec(NamedTuple): +@dataclass(frozen=True, slots=True) +class ColumnSpec: """One column to read, and the type to convert it to. Leave `name` as None to resolve the column by `index` instead. `nullable` decides what a failed @@ -113,13 +95,15 @@ class ColumnSpec(NamedTuple): """ type: ColumnType + _: KW_ONLY name: str | None = None index: int = 0 nullable: bool = False -class OpenOptions(NamedTuple): - """Reader limits and dialect settings for `open_workbook()`/`open_bytes()`. +@dataclass(frozen=True, slots=True, kw_only=True) +class OpenOptions: + """Reader limits and dialect settings for `open_workbook()`. Every field is None for "use the library default", so you only set what you actually want to override. That single convention replaces two the C ABI uses internally: numeric fields there @@ -127,13 +111,15 @@ class OpenOptions(NamedTuple): because several of them default to true. Neither leaks up here. The `max_*` fields are resource limits, not tuning knobs — they bound what a malformed or hostile - file can make the reader allocate. Lower them when parsing untrusted uploads. + file can make the reader allocate. Lower them when parsing untrusted uploads. `max_cell_bytes` + and `intern_strings` apply to every format, a CSV included. `csv_delimiter` and `csv_quote` are byte values, not strings: pass `ord(';')`, not `';'`. - `source_block_size` and `source_cache_bytes` tune the block cache of `open_source()` (4 MiB blocks - in 64 MiB by default; a negative block size turns the cache off). `max_buffered_bytes` caps how - much `open_stream()` reads before an XLSX, XLSB or XLS opens. + `source_block_size` and `source_cache_bytes` tune the block cache of a `Source` passed to + `open_workbook()` (4 MiB blocks in 64 MiB by default; a negative block size turns the cache off). + `max_buffered_bytes` caps how much `open_workbook()` reads from a stream before an XLSX, XLSB or + XLS opens. `verify_zip_checksums` (on by default) checks each ZIP entry of an XLSX or XLSB against its CRC-32, so a damaged file raises instead of returning wrong cells. Set it to `False` to read a @@ -148,8 +134,6 @@ class OpenOptions(NamedTuple): csv_delimiter: int | None = None csv_quote: int | None = None csv_detect_bom: bool | None = None - csv_max_cell_bytes: int | None = None - csv_intern_strings: bool | None = None max_total_decompressed_bytes: int | None = None max_cell_bytes: int | None = None max_shared_string_bytes: int | None = None @@ -162,7 +146,8 @@ class OpenOptions(NamedTuple): verify_zip_checksums: bool | None = None -class WriteOptions(NamedTuple): +@dataclass(frozen=True, slots=True, kw_only=True) +class WriteOptions: """Sheet name and dialect settings for `write_workbook()`. Every field is None for "use the library default", the same convention `OpenOptions` uses. @@ -187,7 +172,7 @@ class StringColumn: """A `ColumnType.STRING` column, held as UTF-8 bytes plus offsets rather than one `str` per row. Decoding is deferred to `column[i]`, so reading a million-row string column costs no Python - objects until something actually asks for a value — the same trade `ColumnarSheet` makes. + objects until something actually asks for a value. """ __slots__ = ("_data", "_offsets") @@ -224,8 +209,9 @@ def __repr__(self) -> str: return f"StringColumn({len(self)} rows, {len(self._data)} bytes)" -class TypedTable(NamedTuple): - """The result of `Sheet.parse_typed()`: one flat buffer per column, already type-converted. +@dataclass(frozen=True, slots=True, kw_only=True) +class TypedTable: + """The result of `Sheet.parse()`: one flat buffer per column, already type-converted. - `names[i]` is column `i`'s name as requested, or its index as a string when the spec resolved by index. @@ -249,10 +235,10 @@ class ExcelReaderError(Exception): class PasswordRequiredError(ExcelReaderError): - """Raised by `open_workbook()`/`open_bytes()` when the workbook is encrypted and no `password` + """Raised by `open_workbook()` when the workbook is encrypted and no `password` was supplied. Maps XL_STATUS_PASSWORD_REQUIRED from the native ABI.""" class PasswordIncorrectError(ExcelReaderError): - """Raised by `open_workbook()`/`open_bytes()` when the supplied `password` did not match the + """Raised by `open_workbook()` when the supplied `password` did not match the workbook's verifier. Maps XL_STATUS_PASSWORD_INCORRECT from the native ABI.""" diff --git a/python/src/excelreader/writer.py b/python/src/excelreader/writer.py index a9ea5db4..57d27c60 100644 --- a/python/src/excelreader/writer.py +++ b/python/src/excelreader/writer.py @@ -1,4 +1,4 @@ -"""Writing a columnar table to a workbook file. Read cursors live in reader.py; this module has none. +"""Writing a columnar table to a workbook file or a binary stream. Read cursors live in reader.py; this module has none. Everything here funnels into the single `xl_write_typed` export. Each column's buffer is copied once into a ctypes block that this module owns for the duration of the call: `xl_write_typed` borrows @@ -11,9 +11,12 @@ import ctypes import datetime +import io +import os +import sys from array import array from pathlib import Path -from typing import Any +from typing import IO, Any, Union from excelreader import _native from excelreader.reader import _check @@ -121,31 +124,65 @@ def _marshal( return specs, native_table, raw_options, keepalive +Destination = Union[str, os.PathLike, IO[bytes]] + + def write_workbook( - path: str | Path, - table: TypedTable, - types: list[ColumnType], + destination: Destination, + data: Any, + types: list[ColumnType] | None = None, *, format: str | None = None, options: WriteOptions | None = None, ) -> None: - """Writes `table` to `path` as a single sheet. + """Writes `data` as a single sheet to `destination`: a path, or a binary object with `write()` + such as `io.BytesIO` or an open file. - `table` is what `Sheet.parse_typed()` returns, so reading a sheet and writing it back out is a - round-trip through the same buffers. `types` gives each column's `ColumnType` and is required - positionally: a `TypedTable` carries raw buffers (an `array`/`StringColumn`/NumPy array), and - nothing about a raw buffer's element size alone distinguishes, say, I64 from TIME — both are - 8-byte-per-row arrays. The type tag has to travel with the call, not be guessed from the data. - `format` is one of xlsx/xlsb/xls/csv; None infers it from the path's extension. + `data` is one of: - `table.names` becomes the header row. `options` sets the sheet name and CSV dialect; see + - a `TypedTable`, what `Sheet.parse()` returns, so reading a sheet and writing it back out + is a round-trip through the same buffers. `types` then gives each column's `ColumnType` and is + required: a `TypedTable` carries raw buffers, and nothing about a raw buffer's element size + alone distinguishes, say, I64 from TIME — both are 8-byte-per-row arrays. + - a `pyarrow.RecordBatch` or `pyarrow.Table`, a `pandas.DataFrame` (its index is not written), + or any other table exposing `__arrow_c_stream__`, such as a `polars.DataFrame`. Each column's + type comes from its Arrow type, so `types` must be None. Requires pyarrow. + + `format` is one of xlsx/xlsb/xls/csv. For a path, None infers it from the extension; an object + with `write()` has no extension, so it needs `format`. The whole workbook is built in memory and + handed to `write()` in one call, at the object's current position; the object is not closed. + + The column names become the header row. `options` sets the sheet name and CSV dialect; see `WriteOptions`. """ - resolved = Path(path) - format_id = _resolve_write_format(format, resolved) - specs, native_table, raw_options, keepalive = _marshal(table, types, options) + if isinstance(data, TypedTable): + if types is None: + raise TypeError("types is required when data is a TypedTable: one ColumnType per column") + table = data + else: + if types is not None: + raise TypeError("types must be None unless data is a TypedTable; Arrow data carries its own types") + table, types = _table_from_arrow(_arrow_table(data)) + if isinstance(destination, (str, os.PathLike)): + _write_file(Path(destination), table, types, format, options) + return + if not hasattr(destination, "write"): + raise TypeError( + f"destination must be a path or an object with write(); got {type(destination).__name__}" + ) + if format is None: + raise ValueError( + f"format= is required when writing to an object with write(); one of {sorted(_FORMAT_NAMES)}" + ) + destination.write(_write_memory(table, types, format, options)) + - encoded = str(resolved).encode("utf-8") +def _write_file( + path: Path, table: TypedTable, types: list[ColumnType], format: str | None, options: WriteOptions | None +) -> None: + format_id = _resolve_write_format(format, path) + specs, native_table, raw_options, keepalive = _marshal(table, types, options) + encoded = str(path).encode("utf-8") lib = _native.load_library() _check( lib.xl_write_typed( @@ -155,32 +192,19 @@ def write_workbook( del keepalive -def write_workbook_to_bytes( - table: TypedTable, - types: list[ColumnType], - *, - format: str, - options: WriteOptions | None = None, +def _write_memory( + table: TypedTable, types: list[ColumnType], format: str, options: WriteOptions | None ) -> bytes: - """The in-memory twin of `write_workbook()`. - - `format` is required and is one of xlsx/xlsb/xls/csv — unlike `write_workbook`, there is no path - to infer it from. - """ - if format not in _native.WRITE_FORMATS: - raise ValueError(f"format must be one of {sorted(_native.WRITE_FORMATS)}; got {format!r}") - + try: + format_id = _FORMAT_NAMES[format.lower()] + except KeyError: + raise ValueError(f"unknown format {format!r}; expected one of {sorted(_FORMAT_NAMES)}") from None specs, native_table, raw_options, keepalive = _marshal(table, types, options) - buffer = _native.NativeBuffer() lib = _native.load_library() _check( lib.xl_write_typed_to_memory( - _native.WRITE_FORMATS[format], - specs, - ctypes.byref(native_table), - ctypes.byref(raw_options), - ctypes.byref(buffer), + format_id, specs, ctypes.byref(native_table), ctypes.byref(raw_options), ctypes.byref(buffer) ) ) try: @@ -195,6 +219,7 @@ def write_workbook_to_bytes( _ARROW_TYPES = { "string": ColumnType.STRING, "large_string": ColumnType.STRING, + "string_view": ColumnType.STRING, "int64": ColumnType.I64, "int32": ColumnType.I64, "double": ColumnType.F64, @@ -314,77 +339,79 @@ def _table_from_arrow(batch: Any) -> tuple[TypedTable, list[ColumnType]]: return table, types -def _arrow_from_pandas(df: Any) -> Any: - import pyarrow - - return pyarrow.Table.from_pandas(df, preserve_index=False) - - -def write_arrow(path: str | Path, batch: Any, **kwargs: Any) -> None: - """Writes a `pyarrow.RecordBatch` (or `Table`) to `path`. Requires pyarrow.""" - table, types = _table_from_arrow(batch) - write_workbook(path, table, types, **kwargs) - - -def write_arrow_to_bytes(batch: Any, *, format: str, options: WriteOptions | None = None) -> bytes: - """The in-memory twin of `write_arrow()`. `format` is required, as in `write_workbook_to_bytes()`.""" - table, types = _table_from_arrow(batch) - return write_workbook_to_bytes(table, types, format=format, options=options) - - -def write_pandas(path: str | Path, df: Any, **kwargs: Any) -> None: - """Writes a `pandas.DataFrame` to `path`. Requires pyarrow and pandas.""" - write_arrow(path, _arrow_from_pandas(df), **kwargs) - - -def write_pandas_to_bytes(df: Any, *, format: str, options: WriteOptions | None = None) -> bytes: - """The in-memory twin of `write_pandas()`. Requires pyarrow and pandas.""" - return write_arrow_to_bytes(_arrow_from_pandas(df), format=format, options=options) - - -def write_polars(path: str | Path, df: Any, **kwargs: Any) -> None: - """Writes a `polars.DataFrame` to `path`. Requires pyarrow and polars.""" - write_arrow(path, df.to_arrow(), **kwargs) - +def _arrow_table(data: Any) -> Any: + try: + import pyarrow + except ImportError: + raise ImportError( + "write_workbook() requires pyarrow to write Arrow or DataFrame data — install it with " + "`pip install pyarrow`, or pass a TypedTable and its types." + ) from None -def write_polars_to_bytes(df: Any, *, format: str, options: WriteOptions | None = None) -> bytes: - """The in-memory twin of `write_polars()`. Requires pyarrow and polars.""" - return write_arrow_to_bytes(df.to_arrow(), format=format, options=options) + if isinstance(data, (pyarrow.RecordBatch, pyarrow.Table)): + return data + pandas = sys.modules.get("pandas") + if pandas is not None and isinstance(data, pandas.DataFrame): + return pyarrow.Table.from_pandas(data, preserve_index=False) + if hasattr(data, "__arrow_c_stream__"): + return pyarrow.RecordBatchReader._import_from_c_capsule(data.__arrow_c_stream__()).read_all() + raise TypeError( + "data must be a TypedTable, a pyarrow RecordBatch or Table, a pandas DataFrame, or an object " + f"with __arrow_c_stream__; got {type(data).__name__}" + ) -def encrypt_package(package_path: str | Path, destination_path: str | Path, password: str) -> None: +def encrypt_package( + package: Union[str, os.PathLike, bytes, bytearray, memoryview, io.BytesIO], + destination: Destination, + password: str, +) -> None: """Wraps a finished plaintext XLSX/XLSB package in an agile-encrypted (ECMA-376 4.4) CFB - container — the inverse of passing `password=` to `open_workbook`/`open_bytes`. `package_path` - is read twice, so it must already be a complete file (write it with `write_workbook` first). - Encryption parameters are fixed at Excel's own defaults; there are no options. + container, the inverse of passing `password=` to `open_workbook`. + + `package` is a path to a complete file, or its bytes (`bytes`, `bytearray`, `memoryview`, or an + `io.BytesIO` read whole). `destination` is a path, or a binary object with `write()` that + receives the encrypted workbook in one call and is not closed. Encryption parameters are fixed + at Excel's own defaults; there are no options. """ - package_encoded = str(Path(package_path)).encode("utf-8") - destination_encoded = str(Path(destination_path)).encode("utf-8") password_encoded = password.encode("utf-8") lib = _native.load_library() - _check( - lib.xl_encrypt_package( - package_encoded, len(package_encoded), - destination_encoded, len(destination_encoded), - password_encoded, len(password_encoded), + if isinstance(package, (str, os.PathLike)) and isinstance(destination, (str, os.PathLike)): + package_encoded = str(Path(package)).encode("utf-8") + destination_encoded = str(Path(destination)).encode("utf-8") + _check( + lib.xl_encrypt_package( + package_encoded, len(package_encoded), + destination_encoded, len(destination_encoded), + password_encoded, len(password_encoded), + ) ) - ) + return + if not isinstance(destination, (str, os.PathLike)) and not hasattr(destination, "write"): + raise TypeError( + f"destination must be a path or an object with write(); got {type(destination).__name__}" + ) + if isinstance(package, (str, os.PathLike)): + data = Path(package).read_bytes() + elif isinstance(package, io.BytesIO): + data = package.getvalue() + elif isinstance(package, (bytes, bytearray, memoryview)): + data = bytes(package) + else: + raise TypeError(f"package must be a path or bytes; got {type(package).__name__}") -def encrypt_package_bytes(package: bytes, password: str) -> bytes: - """The in-memory twin of `encrypt_package()`: takes a finished XLSX/XLSB package's bytes (from - `write_workbook_to_bytes()` or a `*_to_bytes` writer) and returns the encrypted workbook's bytes, - so the plaintext never has to be written to disk. - """ - password_encoded = password.encode("utf-8") buffer = _native.NativeBuffer() - lib = _native.load_library() _check( lib.xl_encrypt_package_to_memory( - package, len(package), password_encoded, len(password_encoded), ctypes.byref(buffer) + data, len(data), password_encoded, len(password_encoded), ctypes.byref(buffer) ) ) try: - return ctypes.string_at(buffer.data, buffer.len) + encrypted = ctypes.string_at(buffer.data, buffer.len) finally: lib.xl_free_buffer(ctypes.byref(buffer)) + if isinstance(destination, (str, os.PathLike)): + Path(destination).write_bytes(encrypted) + else: + destination.write(encrypted) diff --git a/python/src/excelreader/writer_handle.py b/python/src/excelreader/writer_handle.py index 589f9dc6..0356d13b 100644 --- a/python/src/excelreader/writer_handle.py +++ b/python/src/excelreader/writer_handle.py @@ -9,8 +9,9 @@ import ctypes import datetime +import os from pathlib import Path -from typing import Any, Sequence +from typing import IO, Any, Sequence, Union from excelreader import _native from excelreader.reader import _check @@ -33,20 +34,25 @@ def _is_writable(value: Any) -> bool: ) +def _is_typed_blank(value: Any) -> bool: + return type(value) is tuple and len(value) == 2 and value[0] is None and isinstance(value[1], ColumnType) + + class SheetWriter: - """A streaming workbook writer. Use as a context manager; closing finalizes the file. + """A streaming workbook writer. Use as a context manager; closing finalizes the workbook, and for + a destination with `write()` hands it the finished bytes. - The required call order is `start_sheet` -> `start_row` -> one write per cell -> `end_row` -> - `end_sheet`. Calling out of order raises `ExcelReaderError`, and the writer stays usable: fix - the order and continue, or close to discard. + The required call order is `start_sheet` -> `write_row` per row -> `end_sheet`. Calling out of + order raises `ExcelReaderError`, and the writer stays usable: fix the order and continue, or + close to discard. """ - __slots__ = ("_handle", "_lib", "_in_memory") + __slots__ = ("_handle", "_lib", "_destination") - def __init__(self, handle: ctypes.c_void_p, in_memory: bool) -> None: + def __init__(self, handle: ctypes.c_void_p, destination: IO[bytes] | None) -> None: self._handle: ctypes.c_void_p | None = handle self._lib = _native.load_library() - self._in_memory = in_memory + self._destination = destination def _require_handle(self) -> ctypes.c_void_p: if self._handle is None: @@ -57,125 +63,64 @@ def start_sheet(self, name: str) -> None: pointer, length = _encode(name) _check(self._lib.xl_start_sheet(self._require_handle(), pointer, length)) - def start_row(self) -> None: - _check(self._lib.xl_start_row(self._require_handle())) - - def end_row(self) -> None: - _check(self._lib.xl_end_row(self._require_handle())) - def end_sheet(self) -> None: _check(self._lib.xl_end_sheet(self._require_handle())) - def write_str(self, value: str | None) -> None: - """Writes the next cell as text, or a blank cell for None.""" - handle = self._require_handle() - if value is None: - _check(self._lib.xl_write_string(handle, None, 0)) - return - pointer, length = _encode(value) - _check(self._lib.xl_write_string(handle, pointer, length)) - - def write_i64(self, value: int | None) -> None: - """Writes the next cell as a whole number. Raises `OverflowError` outside the signed 64-bit range.""" - if value is None: - self.write_null(ColumnType.I64) - return - # ctypes would keep the low 64 bits of a larger value without complaint. - if not _INT64_MIN <= value <= _INT64_MAX: - raise OverflowError(f"{value} does not fit a 64-bit integer cell; write it with write_f64 or write_str") - _check(self._lib.xl_write_int64(self._require_handle(), value)) - - def write_f64(self, value: float | None) -> None: - if value is None: - self.write_null(ColumnType.F64) - return - _check(self._lib.xl_write_float64(self._require_handle(), value)) - - def write_bool(self, value: bool | None) -> None: - if value is None: - self.write_null(ColumnType.BOOL) - return - _check(self._lib.xl_write_bool(self._require_handle(), 1 if value else 0)) - - def write_date(self, value: datetime.date | None) -> None: - if value is None: - self.write_null(ColumnType.DATE) - return - _check(self._lib.xl_write_date(self._require_handle(), (value - _EPOCH_DATE).days)) - - def write_time(self, value: datetime.time | None) -> None: - if value is None: - self.write_null(ColumnType.TIME) - return - micros = ( - (value.hour * 3600 + value.minute * 60 + value.second) * 1_000_000 + value.microsecond - ) - _check(self._lib.xl_write_time(self._require_handle(), micros)) - - def write_timestamp(self, value: datetime.datetime | None) -> None: - if value is None: - self.write_null(ColumnType.TIMESTAMP) - return - if value.tzinfo is not None: - value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None) - delta = value - _EPOCH_DATETIME - micros = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds - _check(self._lib.xl_write_timestamp(self._require_handle(), micros)) - - def write_null(self, column_type: ColumnType) -> None: - """Writes a blank cell typed as `column_type`.""" - _check(self._lib.xl_write_null(self._require_handle(), int(column_type))) - def write_row(self, values: Sequence[Any]) -> None: - """`start_row()`, one inferred write per value, `end_row()`. + """Writes one row, one cell per value, its type inferred from the value. - The type of each value picks the write method. `None` becomes a blank *string* cell — it is - the one case inference cannot resolve; use `write_null(ColumnType.X)` for a typed blank. + `str` is text, `bool` a boolean, `int` a 64-bit integer, `float` a number, `datetime` a + timestamp (an aware one converted to UTC), `date` a date and `time` a time of day. `None` is a + blank text cell; the tuple `(None, ColumnType.X)` is a blank cell typed as X. Raises, before writing anything, `TypeError` for a value whose type has no mapping and `OverflowError` for an `int` outside the signed 64-bit range. """ for position, value in enumerate(values): - if not _is_writable(value): + if not _is_writable(value) and not _is_typed_blank(value): raise TypeError( f"write_row cannot infer a cell type for {type(value).__name__} " f"at position {position}" ) + # ctypes would keep the low 64 bits of a larger value without complaint. if type(value) is int and not _INT64_MIN <= value <= _INT64_MAX: raise OverflowError(f"{value} at position {position} does not fit a 64-bit integer cell") - self.start_row() + handle = self._require_handle() + _check(self._lib.xl_start_row(handle)) for value in values: - self._write_inferred(value) - self.end_row() + _check(self._write_cell(handle, value)) + _check(self._lib.xl_end_row(handle)) - def _write_inferred(self, value: Any) -> None: + def _write_cell(self, handle: ctypes.c_void_p, value: Any) -> int: + lib = self._lib if value is None: - self.write_str(None) - elif isinstance(value, bool): - self.write_bool(value) - elif isinstance(value, int): - self.write_i64(value) - elif isinstance(value, float): - self.write_f64(value) - elif isinstance(value, datetime.datetime): - self.write_timestamp(value) - elif isinstance(value, datetime.date): - self.write_date(value) - elif isinstance(value, datetime.time): - self.write_time(value) - else: - self.write_str(value) - - def bytes(self) -> bytes: - """The workbook written so far, for an in-memory writer. - - Implicitly finishes the workbook's trailing structure, so it is safe whether or not every - sheet and row was explicitly ended. Unlike `close()`, it does not release the writer — call - `close()` afterward either way. Raises `ExcelReaderError` for a file-backed writer. - """ + return lib.xl_write_string(handle, None, 0) + if isinstance(value, tuple): + return lib.xl_write_null(handle, int(value[1])) + if isinstance(value, bool): + return lib.xl_write_bool(handle, 1 if value else 0) + if isinstance(value, int): + return lib.xl_write_int64(handle, value) + if isinstance(value, float): + return lib.xl_write_float64(handle, value) + if isinstance(value, datetime.datetime): + if value.tzinfo is not None: + value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None) + delta = value - _EPOCH_DATETIME + return lib.xl_write_timestamp(handle, (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds) + if isinstance(value, datetime.date): + return lib.xl_write_date(handle, (value - _EPOCH_DATE).days) + if isinstance(value, datetime.time): + return lib.xl_write_time( + handle, (value.hour * 3600 + value.minute * 60 + value.second) * 1_000_000 + value.microsecond + ) + pointer, length = _encode(value) + return lib.xl_write_string(handle, pointer, length) + + def _finished_bytes(self, handle: ctypes.c_void_p) -> bytes: buffer = _native.NativeBuffer() - _check(self._lib.xl_write_handle_bytes(self._require_handle(), ctypes.byref(buffer))) + _check(self._lib.xl_write_handle_bytes(handle, ctypes.byref(buffer))) try: if not buffer.data or buffer.len <= 0: return b"" @@ -184,11 +129,20 @@ def bytes(self) -> bytes: self._lib.xl_free_buffer(ctypes.byref(buffer)) def close(self) -> None: - """Finalizes and releases the writer. Safe to call more than once.""" + """Finalizes and releases the writer. Safe to call more than once. + + For a destination with `write()`, the finished workbook is written to it in one call, even + when sheets or rows were left open. + """ if self._handle is None: return handle, self._handle = self._handle, None - _check(self._lib.xl_close_write_handle(handle)) + destination, self._destination = self._destination, None + try: + if destination is not None: + destination.write(self._finished_bytes(handle)) + finally: + _check(self._lib.xl_close_write_handle(handle)) def __enter__(self) -> "SheetWriter": return self @@ -204,47 +158,48 @@ def __del__(self) -> None: def open_writer( - path: str | Path, + destination: Union[str, os.PathLike, IO[bytes]], format: str | None = None, + *, options: WriteOptions | None = None, ) -> SheetWriter: - """Opens a streaming writer for `path`, truncating any existing file. + """Opens a streaming writer for `destination`: a path, truncating any existing file, or a binary + object with `write()` such as `io.BytesIO` or an open file. - `format` is one of xlsx/xlsb/xls/csv; None infers it from the path's extension. + `format` is one of xlsx/xlsb/xls/csv. For a path, None infers it from the extension; an object + with `write()` needs it. Such an object receives the whole workbook in one `write()` call when the + writer closes, and is not closed itself. """ from excelreader.writer import _resolve_write_format - resolved = Path(path) - format_id = _resolve_write_format(format, resolved) raw_options = _native.to_native_write_options(options or WriteOptions()) - - encoded = str(resolved).encode("utf-8") - pointer = ctypes.cast(ctypes.c_char_p(encoded), ctypes.POINTER(ctypes.c_uint8)) handle = ctypes.c_void_p() lib = _native.load_library() - _check( - lib.xl_open_write_handle( - pointer, len(encoded), format_id, ctypes.byref(raw_options), ctypes.byref(handle) - ) - ) - return SheetWriter(handle, in_memory=False) + if isinstance(destination, (str, os.PathLike)): + resolved = Path(destination) + format_id = _resolve_write_format(format, resolved) + encoded = str(resolved).encode("utf-8") + pointer = ctypes.cast(ctypes.c_char_p(encoded), ctypes.POINTER(ctypes.c_uint8)) + _check( + lib.xl_open_write_handle( + pointer, len(encoded), format_id, ctypes.byref(raw_options), ctypes.byref(handle) + ) + ) + return SheetWriter(handle, None) -def open_writer_to_memory(format: str, options: WriteOptions | None = None) -> SheetWriter: - """Opens a streaming writer backed by an in-memory buffer, read out with `SheetWriter.bytes()`. - - `format` is one of xlsx/xlsb/xls/csv — unlike `open_writer`, there is no path to infer it from. - """ - if format not in _native.WRITE_FORMATS: - raise ValueError( - f"format must be one of {sorted(_native.WRITE_FORMATS)}; got {format!r}" + if not hasattr(destination, "write"): + raise TypeError( + f"destination must be a path or an object with write(); got {type(destination).__name__}" ) - raw_options = _native.to_native_write_options(options or WriteOptions()) - handle = ctypes.c_void_p() - lib = _native.load_library() - _check( - lib.xl_open_write_handle_to_memory( - _native.WRITE_FORMATS[format], ctypes.byref(raw_options), ctypes.byref(handle) + if format is None: + raise ValueError( + "format= is required when writing to an object with write(); " + f"one of {sorted(_native.WRITE_FORMATS)}" ) - ) - return SheetWriter(handle, in_memory=True) + try: + format_id = _native.WRITE_FORMATS[format.lower()] + except KeyError: + raise ValueError(f"unknown format {format!r}; expected one of {sorted(_native.WRITE_FORMATS)}") from None + _check(lib.xl_open_write_handle_to_memory(format_id, ctypes.byref(raw_options), ctypes.byref(handle))) + return SheetWriter(handle, destination) diff --git a/python/tests/test_reader.py b/python/tests/test_reader.py index 896dbe7f..e5f0be9a 100644 --- a/python/tests/test_reader.py +++ b/python/tests/test_reader.py @@ -16,13 +16,10 @@ OpenOptions, PasswordIncorrectError, PasswordRequiredError, - decode_cell, - open_bytes, open_workbook, ) from excelreader import _native from excelreader import arrow_stream as arrow_stream_module -from excelreader import reader as reader_module ENCRYPTED = Path(__file__).resolve().parents[2] / "tests" / "ExcelReader.Tests" / "data" / "encrypted" @@ -78,7 +75,7 @@ def test_reads_xls(xls_path): def test_reads_from_bytes(xlsx_path): - with open_bytes(xlsx_path.read_bytes()) as workbook: + with open_workbook(xlsx_path.read_bytes()) as workbook: header = next(workbook.sheets[0].rows()) assert header[0].value == "Coluna1" @@ -86,26 +83,18 @@ def test_reads_from_bytes(xlsx_path): def test_exposes_sheet_metadata(xlsx_path): with open_workbook(xlsx_path) as workbook: - assert workbook.sheet_count >= 1 + assert len(workbook.sheets) >= 1 assert workbook.sheets[0].name assert workbook.is_date1904 is False -def test_sheet_names_matches_sheet_count_and_the_sheets(xlsx_path): - with open_workbook(xlsx_path) as workbook: - names = workbook.sheet_names - assert len(names) == workbook.sheet_count == len(workbook.sheets) - assert names[0] == workbook.sheets[0].name - assert [(sheet.index, sheet.name) for sheet in workbook.sheets] == list(enumerate(names)) - - -def test_sheet_names_does_not_disturb_row_enumeration(xlsx_path): +def test_listing_sheet_names_does_not_disturb_row_enumeration(xlsx_path): with open_workbook(xlsx_path) as workbook: rows = workbook.sheets[0].rows() first = next(rows) second = next(rows) - assert workbook.sheet_names + assert [sheet.name for sheet in workbook.sheets] third = next(rows) @@ -123,7 +112,7 @@ def test_use_after_close_raises(xlsx_path): workbook.close() with pytest.raises(ExcelReaderError): - workbook.sheet_count # noqa: B018 + len(workbook.sheets) def test_close_is_idempotent(xlsx_path): @@ -153,7 +142,8 @@ def test_as_date_converts_the_serial_value(xlsx_path): first_data_row = next(rows) date1904 = workbook.is_date1904 - assert first_data_row[1].as_date(date1904) == date(2026, 1, 1) + assert first_data_row[1].date1904 == date1904 + assert first_data_row[1].as_date() == date(2026, 1, 1) def test_as_date_returns_none_for_non_date_cells(xlsx_path): @@ -165,78 +155,12 @@ def test_as_date_returns_none_for_non_date_cells(xlsx_path): assert first_data_row[0].as_date() is None -def test_read_all_matches_row_by_row_iteration(xlsx_path): - with open_workbook(xlsx_path) as workbook: - all_rows = list(workbook.sheets[0].rows()) - - with open_workbook(xlsx_path) as workbook: - bulk_rows = workbook.sheets[0].read_all() - - assert bulk_rows == all_rows - - -def test_read_all_reads_the_whole_sheet_every_time(xlsx_path): +def test_rows_reads_the_whole_sheet_every_time(xlsx_path): with open_workbook(xlsx_path) as workbook: sheet = workbook.sheets[0] - first = sheet.read_all() + first = list(sheet.rows()) assert first - assert sheet.read_all() == first - - -def _columnar_to_rows(sheet) -> list[list[Cell]]: - rows = [] - for row_index in range(len(sheet.row_offsets) - 1): - start, end = sheet.row_offsets[row_index], sheet.row_offsets[row_index + 1] - rows.append([decode_cell(sheet, cell_index) for cell_index in range(start, end)]) - return rows - - -@pytest.mark.parametrize("fixture_name", ["xlsx_path", "xlsb_path"]) -def test_read_all_columnar_matches_read_all(fixture_name, request): - path = request.getfixturevalue(fixture_name) - - with open_workbook(path) as workbook: - expected = workbook.sheets[0].read_all() - - with open_workbook(path) as workbook: - sheet = workbook.sheets[0].read_all_columnar() - - assert _columnar_to_rows(sheet) == expected - - -def test_read_all_columnar_without_numpy(xlsx_path, monkeypatch): - monkeypatch.setattr(reader_module, "_numpy", None) - - with open_workbook(xlsx_path) as workbook: - sheet = workbook.sheets[0].read_all_columnar() - - from array import array - - assert isinstance(sheet.columns, array) - with open_workbook(xlsx_path) as workbook: - assert _columnar_to_rows(sheet) == workbook.sheets[0].read_all() - - -def test_read_all_columnar_grows_the_buffer(xlsx_path, monkeypatch): - monkeypatch.setattr(reader_module, "_INITIAL_ALL_ROWS_BUFFER", 4) - - with open_workbook(xlsx_path) as workbook: - grown = workbook.sheets[0].read_all_columnar() - - with open_workbook(xlsx_path) as workbook: - expected = workbook.sheets[0].read_all() - - assert _columnar_to_rows(grown) == expected - - -def test_read_all_columnar_reads_the_whole_sheet_every_time(xlsx_path): - with open_workbook(xlsx_path) as workbook: - sheet = workbook.sheets[0] - first = sheet.read_all_columnar() - assert len(first.columns) > 0 - assert _columnar_to_rows(sheet.read_all_columnar()) == _columnar_to_rows(first) - - + assert list(sheet.rows()) == first @pytest.fixture @@ -264,7 +188,7 @@ def typed_csv(tmp_path): def test_parse_typed_returns_every_column_type(typed_csv): with open_workbook(typed_csv) as workbook: - table = workbook.sheets[0].parse_typed(_TYPED_SCHEMA) + table = workbook.sheets[0].parse(_TYPED_SCHEMA) assert table.row_count == 2 assert table.names == ["name", "qty", "price", "flag", "day", "clock", "stamp"] @@ -280,7 +204,7 @@ def test_parse_typed_returns_every_column_type(typed_csv): def test_parse_typed_resolves_columns_by_index_when_name_is_none(typed_csv): with open_workbook(typed_csv) as workbook: - table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, index=1)]) + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, index=1)]) assert list(table.columns[0]) == [3, 7] assert table.names == ["1"] @@ -288,7 +212,7 @@ def test_parse_typed_resolves_columns_by_index_when_name_is_none(typed_csv): def test_parse_typed_string_column_supports_len_and_indexing(typed_csv): with open_workbook(typed_csv) as workbook: - table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.STRING, name="name")]) + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.STRING, name="name")]) column = table.columns[0] assert len(column) == 2 @@ -298,7 +222,7 @@ def test_parse_typed_string_column_supports_len_and_indexing(typed_csv): def test_parse_typed_reports_no_validity_bitmap_when_nothing_is_null(typed_csv): with open_workbook(typed_csv) as workbook: - table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, name="qty")]) + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, name="qty")]) assert table.validity == [None] @@ -308,7 +232,7 @@ def test_parse_typed_builds_a_validity_bitmap_for_a_nullable_column(tmp_path): path.write_text("qty\n3\n\n9\n", encoding="utf-8") with open_workbook(path) as workbook: - table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, name="qty", nullable=True)]) + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, name="qty", nullable=True)]) assert table.row_count == 3 assert list(table.columns[0]) == [3, 0, 9] @@ -320,39 +244,38 @@ def test_parse_typed_raises_when_a_non_nullable_column_fails_to_convert(tmp_path path.write_text("qty\n3\nnot-a-number\n", encoding="utf-8") with open_workbook(path) as workbook, pytest.raises(ExcelReaderError): - workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, name="qty")]) + workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, name="qty")]) def test_parse_typed_raises_for_an_unknown_column_name(typed_csv): with open_workbook(typed_csv) as workbook, pytest.raises(ExcelReaderError): - workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, name="nope")]) + workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, name="nope")]) def test_parse_typed_rejects_an_empty_schema(typed_csv): with open_workbook(typed_csv) as workbook, pytest.raises(ValueError): - workbook.sheets[0].parse_typed([]) + workbook.sheets[0].parse([]) def test_parse_typed_reads_the_whole_sheet_regardless_of_the_row_cursor(typed_csv): with open_workbook(typed_csv) as workbook: rows = workbook.sheets[0].rows() next(rows) - table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, name="qty")]) + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, name="qty")]) rest = list(rows) assert table.row_count == 2 assert len(rest) == 2 -def test_to_arrow_returns_a_struct_array_matching_parse_typed(typed_csv): +def test_to_arrow_returns_a_record_batch_matching_parse_typed(typed_csv): pa = pytest.importorskip("pyarrow") with open_workbook(typed_csv) as workbook: - array = workbook.sheets[0].to_arrow(_TYPED_SCHEMA) + batch = workbook.sheets[0].to_arrow(_TYPED_SCHEMA) - assert isinstance(array, pa.StructArray) - assert len(array) == 2 - batch = pa.RecordBatch.from_struct_array(array) + assert isinstance(batch, pa.RecordBatch) + assert batch.num_rows == 2 assert batch.schema.names == ["name", "qty", "price", "flag", "day", "clock", "stamp"] assert batch.column("name").to_pylist() == ["widget", "gadget"] assert batch.column("qty").to_pylist() == [3, 7] @@ -365,10 +288,10 @@ def test_to_arrow_survives_the_workbook_being_closed(typed_csv): pytest.importorskip("pyarrow") with open_workbook(typed_csv) as workbook: - array = workbook.sheets[0].to_arrow([ColumnSpec(ColumnType.I64, name="qty")]) + batch = workbook.sheets[0].to_arrow([ColumnSpec(ColumnType.I64, name="qty")]) gc.collect() - assert array.field(0).to_pylist() == [3, 7] + assert batch.column(0).to_pylist() == [3, 7] def test_to_arrow_raises_for_an_unknown_column_name(typed_csv): @@ -378,17 +301,6 @@ def test_to_arrow_raises_for_an_unknown_column_name(typed_csv): workbook.sheets[0].to_arrow([ColumnSpec(ColumnType.I64, name="nope")]) -def test_to_record_batch_returns_a_column_named_batch_matching_parse_typed(typed_csv): - pa = pytest.importorskip("pyarrow") - - with open_workbook(typed_csv) as workbook: - batch = workbook.sheets[0].to_record_batch(_TYPED_SCHEMA) - - assert isinstance(batch, pa.RecordBatch) - assert batch.schema.names == ["name", "qty", "price", "flag", "day", "clock", "stamp"] - assert batch.column("qty").to_pylist() == [3, 7] - - def test_to_pandas_returns_a_dataframe_matching_parse_typed(typed_csv): pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") @@ -432,10 +344,10 @@ def test_open_options_reaches_the_csv_reader(tmp_path): assert [cell.value for cell in next(workbook.sheets[0].rows())] == ["name", "qty", "price"] -def test_open_options_apply_to_open_bytes_too(tmp_path): +def test_open_options_apply_to_bytes_too(tmp_path): data = b"name;qty\nwidget;3\n" - with open_bytes(data, format="csv", options=OpenOptions(csv_delimiter=ord(";"))) as workbook: + with open_workbook(data, format="csv", options=OpenOptions(csv_delimiter=ord(";"))) as workbook: assert [cell.value for cell in next(workbook.sheets[0].rows())] == ["name", "qty"] @@ -456,15 +368,13 @@ def test_open_options_limit_actually_aborts_an_oversized_read(tmp_path): with open_workbook(path, format="csv") as workbook: assert len(list(workbook.sheets[0].rows())[1][0].value) == 200_000 - with pytest.raises(ExcelReaderError), open_workbook(path, format="csv", options=OpenOptions(csv_max_cell_bytes=100_000)) as workbook: + with pytest.raises(ExcelReaderError), open_workbook(path, format="csv", options=OpenOptions(max_cell_bytes=100_000)) as workbook: list(workbook.sheets[0].rows()) - - def test_opens_encrypted_workbook_with_password(): with open_workbook(ENCRYPTED / "agile-aes256-sha512.xlsx", password="hunter2") as book: - assert book.sheet_count > 0 + assert len(book.sheets) > 0 def test_reports_password_required_without_password(): @@ -479,9 +389,9 @@ def test_reports_password_incorrect_with_wrong_password(): def test_encrypted_rows_match_plaintext(): with open_workbook(ENCRYPTED / "agile-aes256-sha512.xlsx", password="hunter2") as enc: - encrypted_rows = enc.sheets[0].read_all() + encrypted_rows = list(enc.sheets[0].rows()) with open_workbook(ENCRYPTED / "agile-aes256-sha512.plain.xlsx") as plain: - assert plain.sheets[0].read_all() == encrypted_rows + assert list(plain.sheets[0].rows()) == encrypted_rows def test_password_is_not_in_repr(): @@ -511,12 +421,12 @@ def _batch_rows(table): @pytest.mark.parametrize("batch_size", [1, 7, 8, 9, 1000, 0]) -def test_iter_parse_typed_matches_parse_typed(batched_csv, batch_size): +def test_parse_batches_matches_parse_typed(batched_csv, batch_size): with open_workbook(batched_csv) as workbook: - expected = _batch_rows(workbook.sheets[0].parse_typed(_BATCH_SCHEMA)) + expected = _batch_rows(workbook.sheets[0].parse(_BATCH_SCHEMA)) with open_workbook(batched_csv) as workbook: - batches = list(workbook.sheets[0].iter_parse_typed(_BATCH_SCHEMA, batch_size=batch_size)) + batches = list(workbook.sheets[0].parse_batches(_BATCH_SCHEMA, batch_size=batch_size)) assert [row for batch in batches for row in _batch_rows(batch)] == expected @@ -527,17 +437,17 @@ def test_iter_parse_typed_matches_parse_typed(batched_csv, batch_size): assert all(batch.row_count <= batch_size for batch in batches) -def test_iter_parse_typed_rejects_a_negative_batch_size(batched_csv): +def test_parse_batches_rejects_a_negative_batch_size(batched_csv): with open_workbook(batched_csv) as workbook: with pytest.raises(ExcelReaderError): - next(workbook.sheets[0].iter_parse_typed(_BATCH_SCHEMA, batch_size=-1)) + next(workbook.sheets[0].parse_batches(_BATCH_SCHEMA, batch_size=-1)) def test_two_readers_on_one_sheet_each_read_every_row(batched_csv): with open_workbook(batched_csv) as workbook: sheet = workbook.sheets[0] - first = sheet.iter_parse_typed(_BATCH_SCHEMA, batch_size=4) - second = sheet.iter_parse_typed(_BATCH_SCHEMA, batch_size=4) + first = sheet.parse_batches(_BATCH_SCHEMA, batch_size=4) + second = sheet.parse_batches(_BATCH_SCHEMA, batch_size=4) assert next(first).row_count == 4 assert next(second).row_count == 4 @@ -548,9 +458,9 @@ def test_two_readers_on_one_sheet_each_read_every_row(batched_csv): def test_a_whole_sheet_read_leaves_an_open_reader_alone(batched_csv): with open_workbook(batched_csv) as workbook: sheet = workbook.sheets[0] - batches = sheet.iter_parse_typed(_BATCH_SCHEMA, batch_size=4) + batches = sheet.parse_batches(_BATCH_SCHEMA, batch_size=4) next(batches) - assert sheet.parse_typed(_BATCH_SCHEMA).row_count == 50 + assert sheet.parse(_BATCH_SCHEMA).row_count == 50 assert sum(batch.row_count for batch in batches) == 46 @@ -564,7 +474,7 @@ def _spy_on_native_close(monkeypatch, lib, name): def test_abandoning_the_generator_closes_the_reader(batched_csv, monkeypatch): with open_workbook(batched_csv) as workbook: closed = _spy_on_native_close(monkeypatch, workbook._lib, "xl_typed_reader_close") - batches = workbook.sheets[0].iter_parse_typed(_BATCH_SCHEMA, batch_size=4) + batches = workbook.sheets[0].parse_batches(_BATCH_SCHEMA, batch_size=4) next(batches) assert closed == [] @@ -576,28 +486,21 @@ def test_abandoning_the_generator_closes_the_reader(batched_csv, monkeypatch): -def test_to_record_batch_reader_matches_to_record_batch(batched_csv): - pytest.importorskip("pyarrow") +def test_arrow_stream_read_by_pyarrow_matches_to_arrow(batched_csv): + pa = pytest.importorskip("pyarrow") with open_workbook(batched_csv) as workbook: - expected = workbook.sheets[0].to_record_batch(_BATCH_SCHEMA) + expected = workbook.sheets[0].to_arrow(_BATCH_SCHEMA) with open_workbook(batched_csv) as workbook: - reader = workbook.sheets[0].to_record_batch_reader(_BATCH_SCHEMA, batch_size=8) + reader = pa.RecordBatchReader.from_stream(workbook.sheets[0].to_arrow_stream(_BATCH_SCHEMA, batch_size=8)) assert reader.schema == expected.schema batches = list(reader) - assert sum(batch.num_rows for batch in batches) == expected.num_rows + assert pa.Table.from_batches(batches).equals(pa.Table.from_batches([expected])) assert len(batches) == 7, "50 rows at batch size 8 is 7 batches" assert all(batch.num_rows <= 8 for batch in batches) -def test_to_record_batch_reader_rejects_a_negative_batch_size(batched_csv): - pytest.importorskip("pyarrow") - with open_workbook(batched_csv) as workbook: - with pytest.raises(ExcelReaderError): - workbook.sheets[0].to_record_batch_reader(_BATCH_SCHEMA, batch_size=-1) - - def test_arrow_stream_builds_a_polars_dataframe_in_batches(batched_csv): pl = pytest.importorskip("polars") with open_workbook(batched_csv) as workbook: @@ -706,46 +609,28 @@ def test_arrow_stream_rejects_a_negative_batch_size(batched_csv): def test_a_stream_and_a_reader_on_one_sheet_each_read_every_row(batched_csv): - pytest.importorskip("pyarrow") + pa = pytest.importorskip("pyarrow") with open_workbook(batched_csv) as workbook: sheet = workbook.sheets[0] - reader = sheet.iter_parse_typed(_BATCH_SCHEMA, batch_size=4) + reader = sheet.parse_batches(_BATCH_SCHEMA, batch_size=4) next(reader) - stream = sheet.to_record_batch_reader(_BATCH_SCHEMA, batch_size=4) + stream = pa.RecordBatchReader.from_stream(sheet.to_arrow_stream(_BATCH_SCHEMA, batch_size=4)) assert sum(batch.row_count for batch in reader) == 46 assert sum(batch.num_rows for batch in stream) == 50 def test_a_whole_sheet_read_leaves_an_open_stream_alone(batched_csv): - pytest.importorskip("pyarrow") + pa = pytest.importorskip("pyarrow") with open_workbook(batched_csv) as workbook: sheet = workbook.sheets[0] - reader = sheet.to_record_batch_reader(_BATCH_SCHEMA, batch_size=4) + reader = pa.RecordBatchReader.from_stream(sheet.to_arrow_stream(_BATCH_SCHEMA, batch_size=4)) reader.read_next_batch() - assert sheet.parse_typed(_BATCH_SCHEMA).row_count == 50 + assert sheet.parse(_BATCH_SCHEMA).row_count == 50 assert sum(batch.num_rows for batch in reader) == 46 -def test_iter_pandas_yields_one_frame_per_batch(batched_csv): - pytest.importorskip("pyarrow") - pytest.importorskip("pandas") - with open_workbook(batched_csv) as workbook: - frames = list(workbook.sheets[0].iter_pandas(_BATCH_SCHEMA, batch_size=8)) - assert len(frames) == 7 - assert sum(len(frame) for frame in frames) == 50 - - -def test_iter_polars_yields_one_frame_per_batch(batched_csv): - pytest.importorskip("pyarrow") - pytest.importorskip("polars") - with open_workbook(batched_csv) as workbook: - frames = list(workbook.sheets[0].iter_polars(_BATCH_SCHEMA, batch_size=8)) - assert len(frames) == 7 - assert sum(frame.height for frame in frames) == 50 - - def test_to_polars_stays_chunked(batched_csv): polars = pytest.importorskip("polars") with open_workbook(batched_csv) as workbook: @@ -785,8 +670,8 @@ def test_to_pandas_splits_blocks_per_column(batched_csv): def test_parse_typed_in_parallel_matches_the_sequential_table(): with open_workbook(_FIXTURE_CSV) as workbook: - sequential = workbook.sheets[0].parse_typed(_FIXTURE_SCHEMA) - parallel = workbook.sheets[0].parse_typed(_FIXTURE_SCHEMA, parallelism=0) + sequential = workbook.sheets[0].parse(_FIXTURE_SCHEMA) + parallel = workbook.sheets[0].parse(_FIXTURE_SCHEMA, parallelism=0) assert parallel.row_count == sequential.row_count == 65_535 for expected, actual in zip(sequential.columns, parallel.columns): @@ -803,15 +688,25 @@ def test_to_polars_in_parallel_matches_the_streamed_frame(): assert parallel.equals(streamed) +def test_to_pandas_in_parallel_matches_the_streamed_frame(): + pytest.importorskip("pyarrow") + pytest.importorskip("pandas") + with open_workbook(_FIXTURE_CSV) as workbook: + streamed = workbook.sheets[0].to_pandas(_FIXTURE_SCHEMA) + parallel = workbook.sheets[0].to_pandas(_FIXTURE_SCHEMA, parallelism=0) + + assert parallel.equals(streamed) + + def test_parse_typed_rejects_a_negative_parallelism(typed_csv): with open_workbook(typed_csv) as workbook, pytest.raises(ExcelReaderError): - workbook.sheets[0].parse_typed(_TYPED_SCHEMA, parallelism=-1) + workbook.sheets[0].parse(_TYPED_SCHEMA, parallelism=-1) def test_infer_schema_parse_text_types_a_csv(): with open_workbook(_FIXTURE_CSV) as workbook: schema = workbook.sheets[0].infer_schema(parse_text=True) - table = workbook.sheets[0].parse_typed(schema) + table = workbook.sheets[0].parse(schema) T = ColumnType assert [spec.type for spec in schema] == [T.STRING] * 5 + [T.DATE, T.I64, T.DATE, T.I64] + [T.F64] * 5 @@ -840,13 +735,33 @@ def test_infer_schema_without_parse_text_keeps_csv_columns_as_strings(): def test_as_date_follows_excels_calendar(serial, date1904, expected): from datetime import date - assert Cell(0, CellType.DATE, serial).as_date(date1904) == date(*expected) + assert Cell(column=0, type=CellType.DATE, value=serial, date1904=date1904).as_date() == date(*expected) + + +@pytest.mark.parametrize("date1904", [False, True]) +def test_a_cell_read_from_a_workbook_knows_its_date_system(date1904): + import datetime + import io + + from excelreader import WriteOptions, open_writer + + buffer = io.BytesIO() + with open_writer(buffer, "xlsx", options=WriteOptions(date1904=date1904)) as writer: + writer.start_sheet("Data") + writer.write_row([datetime.date(2023, 3, 15)]) + writer.end_sheet() + + with open_workbook(buffer) as workbook: + cell = next(workbook.sheets[0].rows())[0] + + assert cell.date1904 is date1904 + assert cell.as_date() == datetime.date(2023, 3, 15) @pytest.mark.parametrize("serial", ["-1", "2958466", "1e400", "nan"]) def test_as_date_rejects_a_serial_excel_cannot_show(serial): with pytest.raises(ValueError, match="not an Excel date serial"): - Cell(0, CellType.DATE, serial).as_date() + Cell(column=0, type=CellType.DATE, value=serial).as_date() @pytest.mark.parametrize( @@ -862,6 +777,25 @@ def test_parse_typed_timestamp_text_does_not_depend_on_the_local_time_zone(tmp_p path.write_text(f"at\n{text}\n", encoding="utf-8") with open_workbook(path) as workbook: - table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.TIMESTAMP, name="at")]) + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.TIMESTAMP, name="at")]) assert list(table.columns[0]) == [expected_micros] + + +def test_value_types_take_keywords_so_a_new_field_never_shifts_a_caller(): + import io + + from excelreader import WriteOptions, open_writer + + with pytest.raises(TypeError): + OpenOptions(True) + with pytest.raises(TypeError): + WriteOptions("Sheet") + with pytest.raises(TypeError): + ColumnSpec(ColumnType.I64, "qty") + with pytest.raises(TypeError): + Cell(0, CellType.EMPTY, "") + with pytest.raises(TypeError): + tuple(Cell(column=0, type=CellType.EMPTY, value="")) + with pytest.raises(TypeError): + open_writer(io.BytesIO(), "xlsx", WriteOptions()) diff --git a/python/tests/test_sheets.py b/python/tests/test_sheets.py index aa487f06..5677784e 100644 --- a/python/tests/test_sheets.py +++ b/python/tests/test_sheets.py @@ -1,19 +1,21 @@ import gc +import io import threading from concurrent.futures import ThreadPoolExecutor import pytest -from excelreader import ExcelReaderError, Sheet, SheetVisibility, Workbook, open_bytes, open_writer_to_memory +from excelreader import ExcelReaderError, Sheet, SheetVisibility, Workbook, open_workbook, open_writer def _two_sheets() -> bytes: - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: for name, values in (("First", ["a1", "a2"]), ("Second", ["b1", "b2", "b3"])): writer.start_sheet(name) for value in values: writer.write_row([value]) writer.end_sheet() - return writer.bytes() + return buffer.getvalue() def _first_column(sheet: Sheet) -> list[str]: @@ -25,23 +27,24 @@ def _first_column(sheet: Sheet) -> list[str]: def _many_sheets() -> bytes: - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: for sheet in range(_PARALLEL_SHEETS): writer.start_sheet(f"sheet{sheet}") for row in range(_PARALLEL_ROWS): writer.write_row([f"s{sheet}-r{row}", sheet * 1_000_000 + row]) writer.end_sheet() - return writer.bytes() + return buffer.getvalue() def test_a_sheet_reads_its_own_rows(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: assert _first_column(workbook.sheets[1]) == ["b1", "b2", "b3"] assert _first_column(workbook.sheets[0]) == ["a1", "a2"] def test_sheets_is_a_sequence_of_sheets(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: sheets = workbook.sheets assert len(sheets) == 2 @@ -52,19 +55,19 @@ def test_sheets_is_a_sequence_of_sheets(): def test_sheets_looks_a_sheet_up_by_name_without_regard_to_case(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: assert workbook.sheets["Second"].index == 1 assert workbook.sheets["sEcOnD"].index == 1 @pytest.mark.parametrize(("key", "error"), [(5, IndexError), (-3, IndexError), ("nope", KeyError), (1.0, TypeError)]) def test_sheets_rejects_a_key_that_names_no_sheet(key, error): - with open_bytes(_two_sheets()) as workbook, pytest.raises(error): + with open_workbook(_two_sheets()) as workbook, pytest.raises(error): workbook.sheets[key] def test_two_cursors_on_one_sheet_each_return_every_row(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: sheet = workbook.sheets[1] one = sheet.rows() two = sheet.rows() @@ -76,7 +79,7 @@ def test_two_cursors_on_one_sheet_each_return_every_row(): def test_an_abandoned_rows_generator_closes_its_cursor(monkeypatch): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: sheet = workbook.sheets[1] closed = [] real_close = workbook._lib.xl_rows_close @@ -94,7 +97,7 @@ def test_an_abandoned_rows_generator_closes_its_cursor(monkeypatch): def test_a_sheet_of_a_closed_workbook_raises(): - workbook = open_bytes(_two_sheets()) + workbook = open_workbook(_two_sheets()) sheet = workbook.sheets[0] workbook.close() @@ -105,7 +108,7 @@ def test_a_sheet_of_a_closed_workbook_raises(): def test_rows_opened_before_close_read_to_the_end_after_it(): - workbook = open_bytes(_two_sheets()) + workbook = open_workbook(_two_sheets()) rows = workbook.sheets[1].rows() assert next(rows)[0].value == "b1" @@ -114,22 +117,20 @@ def test_rows_opened_before_close_read_to_the_end_after_it(): assert [row[0].value for row in rows] == ["b2", "b3"] -def test_read_all_and_parse_typed_read_the_named_sheet(): +def test_typed_reads_read_the_named_sheet(): from excelreader import ColumnSpec, ColumnType - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: second = workbook.sheets[1] schema = [ColumnSpec(ColumnType.STRING, index=0, nullable=True)] - assert len(second.read_all()) == 3 - assert len(second.read_all_columnar().row_offsets) == 4 - assert second.parse_typed(schema, header_row=0).row_count == 3 - assert sum(table.row_count for table in second.iter_parse_typed(schema, header_row=0, batch_size=2)) == 3 + assert second.parse(schema, header_row=0).row_count == 3 + assert sum(table.row_count for table in second.parse_batches(schema, header_row=0, batch_size=2)) == 3 assert len(second.infer_schema(header_row=0)) == 1 def test_every_sheet_read_on_its_own_thread_matches_a_sequential_read(): - with open_bytes(_many_sheets()) as workbook: + with open_workbook(_many_sheets()) as workbook: sheets = list(workbook.sheets) sequential = [_first_column(sheet) for sheet in sheets] assert sequential[3][0] == "s3-r0" @@ -146,23 +147,23 @@ def test_typed_parses_of_every_sheet_run_on_separate_threads(): from excelreader import ColumnSpec, ColumnType schema = [ColumnSpec(ColumnType.I64, index=1)] - with open_bytes(_many_sheets()) as workbook, ThreadPoolExecutor(max_workers=_PARALLEL_SHEETS) as pool: - tables = list(pool.map(lambda sheet: sheet.parse_typed(schema, header_row=0), workbook.sheets)) + with open_workbook(_many_sheets()) as workbook, ThreadPoolExecutor(max_workers=_PARALLEL_SHEETS) as pool: + tables = list(pool.map(lambda sheet: sheet.parse(schema, header_row=0), workbook.sheets)) for sheet, table in enumerate(tables): assert table.row_count == _PARALLEL_ROWS assert int(table.columns[0][0]) == sheet * 1_000_000 -def test_a_record_batch_reader_outlives_its_workbook(): +def test_an_arrow_stream_outlives_its_workbook(): pyarrow = pytest.importorskip("pyarrow") from excelreader import ColumnSpec, ColumnType schema = [ColumnSpec(ColumnType.I64, index=1)] - with open_bytes(_many_sheets()) as workbook: - reader = workbook.sheets[2].to_record_batch_reader(schema, header_row=0, batch_size=100) + with open_workbook(_many_sheets()) as workbook: + stream = workbook.sheets[2].to_arrow_stream(schema, header_row=0, batch_size=100) - table = reader.read_all() + table = pyarrow.RecordBatchReader.from_stream(stream).read_all() assert isinstance(table, pyarrow.Table) assert table.num_rows == _PARALLEL_ROWS @@ -171,7 +172,7 @@ def test_a_record_batch_reader_outlives_its_workbook(): def test_sheets_compare_by_workbook_and_index(): data = _two_sheets() - with open_bytes(data) as workbook, open_bytes(data) as other: + with open_workbook(data) as workbook, open_workbook(data) as other: sheets = workbook.sheets assert sheets.index(sheets[1]) == 1 assert sheets[0] in sheets @@ -182,18 +183,14 @@ def test_sheets_compare_by_workbook_and_index(): def test_a_sheet_with_no_rows_reads_empty(): - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: writer.start_sheet("Empty") writer.end_sheet() - data = writer.bytes() + data = buffer.getvalue() - with open_bytes(data) as workbook: - sheet = workbook.sheets[0] - assert sheet.read_all() == [] - columnar = sheet.read_all_columnar() - assert list(columnar.row_offsets) == [0] - assert columnar.values == b"" - assert list(sheet.rows()) == [] + with open_workbook(data) as workbook: + assert list(workbook.sheets[0].rows()) == [] class _SlowCheckWorkbook(Workbook): @@ -219,7 +216,7 @@ def _handle(self, value): def test_closing_from_two_threads_at_once_raises_nowhere(): - source = open_bytes(_two_sheets()) + source = open_workbook(_two_sheets()) workbook = _SlowCheckWorkbook(source._handle) source._handle = None errors: list[BaseException] = [] diff --git a/python/tests/test_sources.py b/python/tests/test_sources.py index a8353732..209c49ef 100644 --- a/python/tests/test_sources.py +++ b/python/tests/test_sources.py @@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import pytest -from excelreader import ExcelReaderError, OpenOptions, open_bytes, open_source, open_stream, open_writer_to_memory +from excelreader import ExcelReaderError, OpenOptions, open_workbook, open_writer from excelreader import sources as sources_module SHEETS = 4 @@ -14,13 +14,14 @@ def _workbook_bytes() -> bytes: - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: for sheet in range(SHEETS): writer.start_sheet(f"sheet{sheet}") for row in range(ROWS): writer.write_row([f"s{sheet}-r{row}"]) writer.end_sheet() - return writer.bytes() + return buffer.getvalue() def _first_column(sheet) -> list[str]: @@ -59,10 +60,10 @@ def read(self, size: int) -> bytes: def test_every_sheet_of_a_source_reads_in_parallel_like_memory(): data = _workbook_bytes() - with open_bytes(data) as reference: + with open_workbook(data) as reference: expected = [_first_column(sheet) for sheet in reference.sheets] - with open_source(BytesSource(data)) as workbook, ThreadPoolExecutor(max_workers=SHEETS * 2) as pool: + with open_workbook(BytesSource(data)) as workbook, ThreadPoolExecutor(max_workers=SHEETS * 2) as pool: sheets = list(workbook.sheets) parallel = list(pool.map(_first_column, sheets + sheets)) @@ -74,7 +75,7 @@ def test_the_source_is_dropped_after_the_workbook_closes(): source = BytesSource(_workbook_bytes()) finalized = weakref.finalize(source, lambda: None) before = set(sources_module._live) - workbook = open_source(source, "xlsx") + workbook = open_workbook(source, "xlsx") assert len(sources_module._live) == len(before) + 1 del source _first_column(workbook.sheets[0]) @@ -86,32 +87,32 @@ def test_the_source_is_dropped_after_the_workbook_closes(): def test_a_source_error_reaches_the_caller_with_its_message(): with pytest.raises(ExcelReaderError, match="disk on fire"): - open_source(FailingSource(), "xlsx") + open_workbook(FailingSource(), "xlsx") def test_the_default_cache_fetches_a_small_workbook_once(): data = _workbook_bytes() cached = BytesSource(data) - with open_source(cached, "xlsx") as workbook: + with open_workbook(cached, "xlsx") as workbook: _first_column(workbook.sheets[0]) assert cached.reads == 1 uncached = BytesSource(data) - with open_source(uncached, "xlsx", options=OpenOptions(source_block_size=-1)) as workbook: + with open_workbook(uncached, "xlsx", options=OpenOptions(source_block_size=-1)) as workbook: _first_column(workbook.sheets[0]) assert uncached.reads > 1 def test_one_source_object_opened_twice_at_once_serves_both(): source = BytesSource(_workbook_bytes()) - first = open_source(source, "xlsx") - second = open_source(source, "xlsx") + first = open_workbook(source, "xlsx") + second = open_workbook(source, "xlsx") first.close() assert _first_column(second.sheets[2])[0] == "s2-r0" second.close() -def test_open_stream_reads_a_csv_from_a_pipe(): +def test_a_stream_reads_a_csv_from_a_pipe(): read_fd, write_fd = os.pipe() def produce() -> None: @@ -122,7 +123,7 @@ def produce() -> None: producer = threading.Thread(target=produce) producer.start() - with os.fdopen(read_fd, "rb") as pipe, open_stream(pipe, "csv") as workbook: + with os.fdopen(read_fd, "rb") as pipe, open_workbook(pipe, "csv") as workbook: names = _first_column(workbook.sheets[0]) producer.join() assert names[0] == "name" @@ -132,7 +133,7 @@ def produce() -> None: def test_a_csv_stream_serves_a_schema_inference_and_then_every_row(): data = b"name,qty\n" + b"".join(f"row{i},{i}\n".encode() for i in range(5000)) - with open_stream(OnlyRead(data), "csv") as workbook: + with open_workbook(OnlyRead(data), "csv") as workbook: sheet = workbook.sheets[0] schema = sheet.infer_schema(parse_text=True) names = _first_column(sheet) @@ -141,59 +142,94 @@ def test_a_csv_stream_serves_a_schema_inference_and_then_every_row(): assert names[-1] == "row4999" -def test_open_stream_reads_an_xlsx_from_an_object_with_only_read(): - with open_stream(OnlyRead(_workbook_bytes())) as workbook: +def test_a_stream_reads_an_xlsx_from_an_object_with_only_read(): + with open_workbook(OnlyRead(_workbook_bytes())) as workbook: rows = _first_column(workbook.sheets[3]) assert rows[0] == "s3-r0" assert len(rows) == ROWS -def test_open_stream_rejects_an_object_that_cannot_be_read(): - with pytest.raises(TypeError): - open_stream(object()) +def test_open_workbook_rejects_an_object_it_cannot_read(): + with pytest.raises(TypeError, match="object"): + open_workbook(object()) + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda data, path: str(path), id="str"), + pytest.param(lambda data, path: path, id="path"), + pytest.param(lambda data, path: data, id="bytes"), + pytest.param(lambda data, path: bytearray(data), id="bytearray"), + pytest.param(lambda data, path: memoryview(data), id="memoryview"), + pytest.param(lambda data, path: io.BytesIO(data), id="bytesio"), + pytest.param(lambda data, path: BytesSource(data), id="source"), + pytest.param(lambda data, path: OnlyRead(data), id="stream"), + pytest.param(lambda data, path: path.open("rb"), id="file"), + ], +) +def test_open_workbook_reads_the_same_rows_from_every_kind_of_input(tmp_path, wrap): + data = _workbook_bytes() + path = tmp_path / "book.xlsx" + path.write_bytes(data) + source = wrap(data, path) + try: + with open_workbook(source) as workbook: + rows = [_first_column(sheet) for sheet in workbook.sheets] + finally: + if hasattr(source, "close"): + source.close() + assert rows == [[f"s{sheet}-r{row}" for row in range(ROWS)] for sheet in range(SHEETS)] + + +def test_a_bytesio_is_read_from_its_start_whatever_its_position(): + buffer = io.BytesIO(_workbook_bytes()) + buffer.seek(0, io.SEEK_END) + with open_workbook(buffer) as workbook: + assert _first_column(workbook.sheets[0])[0] == "s0-r0" def test_a_failed_open_leaves_nothing_registered(): before = set(sources_module._live) source = BytesSource(_workbook_bytes()) with pytest.raises(ValueError): - open_source(source, "nope") + open_workbook(source, "nope") with pytest.raises(ValueError): - open_source(source, "xlsx", password="\ud800") + open_workbook(source, "xlsx", password="\ud800") negative = BytesSource(b"") negative.size = -1 with pytest.raises(ValueError): - open_source(negative, "xlsx") + open_workbook(negative, "xlsx") with pytest.raises(ValueError): - open_stream(io.BytesIO(), "nope") + open_workbook(OnlyRead(b""), "nope") assert set(sources_module._live) == before -def test_open_stream_fails_on_a_text_stream_and_releases_it(): +def test_a_stream_fails_on_a_text_stream_and_releases_it(): before = set(sources_module._live) - with pytest.raises(ExcelReaderError), open_stream(io.StringIO("a,b\n1,2\n"), "csv") as workbook: + with pytest.raises(ExcelReaderError), open_workbook(io.StringIO("a,b\n1,2\n"), "csv") as workbook: _first_column(workbook.sheets[0]) assert set(sources_module._live) == before -def test_open_stream_fails_when_read_returns_more_than_asked(): +def test_a_stream_fails_when_read_returns_more_than_asked(): class Greedy: def read(self, size: int) -> bytes: return b"x" * (size + 1) before = set(sources_module._live) - with pytest.raises(ExcelReaderError, match="more bytes than requested"), open_stream(Greedy(), "csv") as workbook: + with pytest.raises(ExcelReaderError, match="more bytes than requested"), open_workbook(Greedy(), "csv") as workbook: _first_column(workbook.sheets[0]) assert set(sources_module._live) == before -def test_open_stream_fails_when_read_raises_keyboard_interrupt_and_releases_it(): +def test_a_stream_fails_when_read_raises_keyboard_interrupt_and_releases_it(): class Interrupted: def read(self, size: int) -> bytes: raise KeyboardInterrupt before = set(sources_module._live) - with pytest.raises(ExcelReaderError), open_stream(Interrupted(), "csv") as workbook: + with pytest.raises(ExcelReaderError), open_workbook(Interrupted(), "csv") as workbook: _first_column(workbook.sheets[0]) assert set(sources_module._live) == before @@ -206,4 +242,4 @@ def read_at(self, offset: int, buffer: memoryview): return None with pytest.raises(ExcelReaderError): - open_source(Bad(), "xlsx") + open_workbook(Bad(), "xlsx") diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index de2da3dc..b7629cf7 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -1,3 +1,5 @@ +import io + import pytest from excelreader import ( ColumnSpec, @@ -6,17 +8,8 @@ PasswordIncorrectError, WriteOptions, encrypt_package, - encrypt_package_bytes, - open_bytes, open_workbook, - write_arrow, - write_arrow_to_bytes, - write_pandas, - write_pandas_to_bytes, - write_polars, - write_polars_to_bytes, write_workbook, - write_workbook_to_bytes, ) _SCHEMA = [ @@ -36,13 +29,13 @@ def source_csv(tmp_path): @pytest.mark.parametrize("extension", ["xlsx", "xlsb", "xls", "csv"]) def test_write_workbook_round_trips_every_format(source_csv, tmp_path, extension): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) out = tmp_path / f"out.{extension}" write_workbook(out, table, _TYPES) with open_workbook(out) as workbook: - result = workbook.sheets[0].parse_typed(_SCHEMA) + result = workbook.sheets[0].parse(_SCHEMA) assert result.row_count == 2 assert list(result.columns[0]) == ["widget", "gadget"] @@ -55,13 +48,13 @@ def test_write_workbook_round_trips_a_nullable_column(tmp_path): schema = [ColumnSpec(ColumnType.I64, name="qty", nullable=True)] with open_workbook(source) as workbook: - table = workbook.sheets[0].parse_typed(schema) + table = workbook.sheets[0].parse(schema) out = tmp_path / "out.xlsx" write_workbook(out, table, [spec.type for spec in schema]) with open_workbook(out) as workbook: - result = workbook.sheets[0].parse_typed(schema) + result = workbook.sheets[0].parse(schema) assert result.row_count == 3 assert result.validity[0][0] & 0b111 == 0b101 @@ -69,18 +62,18 @@ def test_write_workbook_round_trips_a_nullable_column(tmp_path): def test_write_workbook_applies_the_sheet_name(source_csv, tmp_path): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) out = tmp_path / "named.xlsx" write_workbook(out, table, _TYPES, options=WriteOptions(sheet_name="Vendas")) with open_workbook(out) as workbook: - assert workbook.sheet_names == ["Vendas"] + assert [sheet.name for sheet in workbook.sheets] == ["Vendas"] def test_write_workbook_rejects_an_unknown_extension(source_csv, tmp_path): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) with pytest.raises(ValueError, match="format"): write_workbook(tmp_path / "out.parquet", table, _TYPES) @@ -88,41 +81,82 @@ def test_write_workbook_rejects_an_unknown_extension(source_csv, tmp_path): def test_write_workbook_reports_a_native_rejection(source_csv, tmp_path): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) with pytest.raises(ExcelReaderError): write_workbook(tmp_path / "out.xlsx", table, _TYPES, options=WriteOptions(sheet_name="has/slash")) -def test_write_arrow_round_trips(tmp_path): +def test_write_workbook_rejects_a_typed_table_without_types(source_csv, tmp_path): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse(_SCHEMA) + + with pytest.raises(TypeError, match="types"): + write_workbook(tmp_path / "out.xlsx", table) + + +def test_write_workbook_rejects_types_for_arrow_data(tmp_path): pa = pytest.importorskip("pyarrow") - batch = pa.RecordBatch.from_pydict({"name": ["widget", "gadget"], "qty": [3, 7]}) + batch = pa.RecordBatch.from_pydict({"name": ["widget"], "qty": [3]}) + with pytest.raises(TypeError, match="types"): + write_workbook(tmp_path / "out.xlsx", batch, _TYPES) + + +def test_write_workbook_rejects_data_it_cannot_write(tmp_path): + pytest.importorskip("pyarrow") + + with pytest.raises(TypeError, match="data"): + write_workbook(tmp_path / "out.xlsx", [["widget", 3]]) + + +@pytest.mark.parametrize("kind", ["record_batch", "table"]) +def test_write_workbook_writes_arrow_data(tmp_path, kind): + pa = pytest.importorskip("pyarrow") + + data = {"name": ["widget", "gadget"], "qty": [3, 7]} + frame = pa.RecordBatch.from_pydict(data) if kind == "record_batch" else pa.table(data) out = tmp_path / "arrow.xlsx" - write_arrow(out, batch) + write_workbook(out, frame) with open_workbook(out) as workbook: - result = workbook.sheets[0].parse_typed(_SCHEMA) + result = workbook.sheets[0].parse(_SCHEMA) assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] -def test_write_pandas_round_trips(tmp_path): +def test_write_workbook_leaves_a_pandas_index_out(tmp_path): pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") out = tmp_path / "pandas.xlsx" - write_pandas(out, pd.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]})) + write_workbook(out, pd.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]}, index=[10, 20])) with open_workbook(out) as workbook: - result = workbook.sheets[0].parse_typed(_SCHEMA) + rows = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] + + assert rows == [["name", "qty"], ["widget", "3"], ["gadget", "7"]] + + +def test_write_workbook_writes_any_arrow_c_stream(tmp_path): + pa = pytest.importorskip("pyarrow") + + class Producer: + def __arrow_c_stream__(self, requested_schema=None): + return pa.table({"name": ["widget", "gadget"], "qty": [3, 7]}).__arrow_c_stream__(requested_schema) + + out = tmp_path / "stream.xlsx" + write_workbook(out, Producer()) + + with open_workbook(out) as workbook: + result = workbook.sheets[0].parse(_SCHEMA) assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] -def test_write_pandas_round_trips_a_multi_chunk_frame(tmp_path): +def test_write_workbook_round_trips_a_multi_chunk_pandas_frame(tmp_path): pa = pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") @@ -136,58 +170,58 @@ def test_write_pandas_round_trips_a_multi_chunk_frame(tmp_path): assert pa.Table.from_pandas(frame, preserve_index=False).column(0).num_chunks > 1 out = tmp_path / "chunked.xlsx" - write_pandas(out, frame) + write_workbook(out, frame) with open_workbook(out) as workbook: - result = workbook.sheets[0].parse_typed(_SCHEMA) + result = workbook.sheets[0].parse(_SCHEMA) assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] -def test_write_pandas_writes_a_header_only_sheet_for_an_empty_frame(tmp_path): +def test_write_workbook_writes_a_header_only_sheet_for_an_empty_frame(tmp_path): pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") out = tmp_path / "empty.xlsx" - write_pandas( + write_workbook( out, pd.DataFrame({"name": pd.Series([], dtype="string"), "qty": pd.Series([], dtype="int64")}), ) with open_workbook(out) as workbook: - rows = workbook.sheets[0].read_all() + rows = list(workbook.sheets[0].rows()) assert [cell.value for cell in rows[0]] == ["name", "qty"] assert len(rows) == 1 -def test_write_polars_round_trips(tmp_path): +def test_write_workbook_writes_a_polars_frame(tmp_path): pytest.importorskip("pyarrow") pl = pytest.importorskip("polars") out = tmp_path / "polars.xlsx" - write_polars(out, pl.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]})) + write_workbook(out, pl.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]})) with open_workbook(out) as workbook: - result = workbook.sheets[0].parse_typed(_SCHEMA) + result = workbook.sheets[0].parse(_SCHEMA) assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] -def test_write_arrow_rejects_an_unsupported_arrow_type(tmp_path): +def test_write_workbook_rejects_an_unsupported_arrow_type(tmp_path): pa = pytest.importorskip("pyarrow") batch = pa.RecordBatch.from_pydict({"nested": [[1, 2], [3]]}) with pytest.raises(ValueError, match="type"): - write_arrow(tmp_path / "bad.xlsx", batch) + write_workbook(tmp_path / "bad.xlsx", batch) def test_encrypt_package_round_trips_through_open_workbook(source_csv, tmp_path): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) plain = tmp_path / "plain.xlsx" write_workbook(plain, table, _TYPES) @@ -196,7 +230,7 @@ def test_encrypt_package_round_trips_through_open_workbook(source_csv, tmp_path) encrypt_package(plain, encrypted, "hunter2") with open_workbook(encrypted, password="hunter2") as workbook: - result = workbook.sheets[0].parse_typed(_SCHEMA) + result = workbook.sheets[0].parse(_SCHEMA) assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] @@ -204,7 +238,7 @@ def test_encrypt_package_round_trips_through_open_workbook(source_csv, tmp_path) def test_encrypt_package_rejects_the_wrong_password_on_open(source_csv, tmp_path): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) plain = tmp_path / "plain.xlsx" write_workbook(plain, table, _TYPES) @@ -218,7 +252,7 @@ def test_encrypt_package_rejects_the_wrong_password_on_open(source_csv, tmp_path def test_encrypt_package_rejects_an_empty_password(source_csv, tmp_path): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) plain = tmp_path / "plain.xlsx" write_workbook(plain, table, _TYPES) @@ -228,15 +262,18 @@ def test_encrypt_package_rejects_an_empty_password(source_csv, tmp_path): def _parse_bytes(payload, format, password=None): - with open_bytes(payload, format=format, password=password) as workbook: - return workbook.sheets[0].parse_typed(_SCHEMA) + with open_workbook(payload, format=format, password=password) as workbook: + return workbook.sheets[0].parse(_SCHEMA) -@pytest.mark.parametrize( - ("writer", "frame_module"), - [(write_arrow_to_bytes, None), (write_pandas_to_bytes, "pandas"), (write_polars_to_bytes, "polars")], -) -def test_frame_writers_to_bytes_round_trip(writer, frame_module): +def _to_bytes(table): + buffer = io.BytesIO() + write_workbook(buffer, table, _TYPES, format="xlsx") + return buffer.getvalue() + + +@pytest.mark.parametrize("frame_module", [None, "pandas", "polars"]) +def test_frames_round_trip_through_a_bytesio(frame_module): pa = pytest.importorskip("pyarrow") data = {"name": ["widget", "gadget"], "qty": [3, 7]} if frame_module is None: @@ -244,33 +281,83 @@ def test_frame_writers_to_bytes_round_trip(writer, frame_module): else: frame = pytest.importorskip(frame_module).DataFrame(data) - result = _parse_bytes(writer(frame, format="xlsx", options=WriteOptions(sheet_name="Vendas")), "xlsx") + buffer = io.BytesIO() + write_workbook(buffer, frame, format="xlsx", options=WriteOptions(sheet_name="Vendas")) + result = _parse_bytes(buffer.getvalue(), "xlsx") assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] -def test_encrypt_package_bytes_round_trips_without_touching_disk(source_csv): +@pytest.mark.parametrize("package_kind", ["path", "bytes", "bytesio"]) +@pytest.mark.parametrize("destination_kind", ["path", "bytesio"]) +def test_encrypt_package_takes_paths_and_buffers_on_both_sides( + tmp_path, source_csv, package_kind, destination_kind +): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) + plain = _to_bytes(table) + plain_path = tmp_path / "plain.xlsx" + plain_path.write_bytes(plain) + package = {"path": plain_path, "bytes": plain, "bytesio": io.BytesIO(plain)}[package_kind] + destination = tmp_path / "secret.xlsx" if destination_kind == "path" else io.BytesIO() - encrypted = encrypt_package_bytes(write_workbook_to_bytes(table, _TYPES, format="xlsx"), "hunter2") - result = _parse_bytes(encrypted, "xlsx", password="hunter2") + encrypt_package(package, destination, "hunter2") + encrypted = destination.read_bytes() if destination_kind == "path" else destination.getvalue() + result = _parse_bytes(encrypted, "xlsx", password="hunter2") assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] with pytest.raises(PasswordIncorrectError): - open_bytes(encrypted, format="xlsx", password="wrong") + open_workbook(encrypted, format="xlsx", password="wrong") -def test_encrypt_package_bytes_rejects_bytes_that_are_not_a_package(): +def test_encrypt_package_rejects_bytes_that_are_not_a_package(): with pytest.raises(ExcelReaderError): - encrypt_package_bytes(b"not a zip", "hunter2") + encrypt_package(b"not a zip", io.BytesIO(), "hunter2") -def test_encrypt_package_bytes_rejects_an_empty_password(source_csv): +def test_encrypt_package_rejects_an_empty_password_in_memory(source_csv): with open_workbook(source_csv) as workbook: - table = workbook.sheets[0].parse_typed(_SCHEMA) + table = workbook.sheets[0].parse(_SCHEMA) with pytest.raises(ExcelReaderError): - encrypt_package_bytes(write_workbook_to_bytes(table, _TYPES, format="xlsx"), "") + encrypt_package(_to_bytes(table), io.BytesIO(), "") + + +def test_encrypt_package_rejects_inputs_it_cannot_use(): + with pytest.raises(TypeError, match="package"): + encrypt_package(42, io.BytesIO(), "hunter2") + with pytest.raises(TypeError, match="destination"): + encrypt_package(b"x", 42, "hunter2") + + +def test_write_workbook_writes_into_an_open_binary_file(tmp_path, source_csv): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse(_SCHEMA) + path = tmp_path / "out.bin" + + with path.open("wb") as file: + write_workbook(file, table, _TYPES, format="xlsb") + + result = _parse_bytes(path.read_bytes(), "xlsb") + assert list(result.columns[0]) == ["widget", "gadget"] + assert list(result.columns[1]) == [3, 7] + + +def test_write_workbook_to_a_stream_needs_a_format(source_csv): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse(_SCHEMA) + + with pytest.raises(ValueError, match="format"): + write_workbook(io.BytesIO(), table, _TYPES) + with pytest.raises(ValueError, match="unknown format"): + write_workbook(io.BytesIO(), table, _TYPES, format="parquet") + + +def test_write_workbook_rejects_a_destination_it_cannot_write(source_csv): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse(_SCHEMA) + + with pytest.raises(TypeError, match="destination"): + write_workbook(42, table, _TYPES, format="xlsx") diff --git a/python/tests/test_writer_handle.py b/python/tests/test_writer_handle.py index 7b3b6042..cbb95378 100644 --- a/python/tests/test_writer_handle.py +++ b/python/tests/test_writer_handle.py @@ -4,21 +4,15 @@ import pytest -from excelreader import CellType, ColumnType, open_workbook, open_writer +from excelreader import CellType, ColumnSpec, ColumnType, open_workbook, open_writer def test_round_trips_typed_cells(tmp_path): path = tmp_path / "out.xlsx" with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str("name") - writer.write_str("qty") - writer.end_row() - writer.start_row() - writer.write_str("widget") - writer.write_i64(7) - writer.end_row() + writer.write_row(["name", "qty"]) + writer.write_row(["widget", 7]) writer.end_sheet() with open_workbook(path) as workbook: @@ -33,7 +27,7 @@ def test_round_trips_typed_cells(tmp_path): strict=True, reason=( "native reader bug: XlsxReader.Enumerator.cs ParseCellSpan skips self-closing elements " - "instead of recording them as blank cells, so a trailing blank cell written by write_null is " + "instead of recording them as blank cells, so a trailing typed blank cell is " "missing from the decoded row (len 7, not 8). Pre-existing in the native core, out of scope " "for Task 6 (Python bindings only) - remove this marker once the native reader is fixed." ), @@ -42,16 +36,18 @@ def test_writes_every_scalar_type(tmp_path): path = tmp_path / "types.xlsx" with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str("s") - writer.write_i64(1) - writer.write_f64(1.5) - writer.write_bool(True) - writer.write_date(datetime.date(2020, 3, 4)) - writer.write_time(datetime.time(13, 45, 6)) - writer.write_timestamp(datetime.datetime(2020, 3, 4, 13, 45, 6)) - writer.write_null(ColumnType.I64) - writer.end_row() + writer.write_row( + [ + "s", + 1, + 1.5, + True, + datetime.date(2020, 3, 4), + datetime.time(13, 45, 6), + datetime.datetime(2020, 3, 4, 13, 45, 6), + (None, ColumnType.I64), + ] + ) writer.end_sheet() with open_workbook(path) as workbook: @@ -62,14 +58,11 @@ def test_writes_every_scalar_type(tmp_path): assert len(rows[0]) == 8 -def test_write_str_none_is_a_blank_cell(tmp_path): +def test_none_is_a_blank_cell(tmp_path): path = tmp_path / "blank.xlsx" with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str(None) - writer.write_str("after") - writer.end_row() + writer.write_row([None, "after"]) writer.end_sheet() with open_workbook(path) as workbook: @@ -78,16 +71,42 @@ def test_write_str_none_is_a_blank_cell(tmp_path): assert rows[0][-1] == "after" +def test_a_typed_blank_reads_back_as_a_null_of_that_type(tmp_path): + path = tmp_path / "typed-blank.xlsx" + with open_writer(path) as writer: + writer.start_sheet("Data") + writer.write_row(["qty"]) + writer.write_row([(None, ColumnType.I64)]) + writer.write_row([5]) + writer.end_sheet() + + with open_workbook(path) as workbook: + assert workbook.sheets[0].infer_schema() == [ColumnSpec(ColumnType.I64, name="qty", nullable=True)] + table = workbook.sheets[0].parse([ColumnSpec(ColumnType.I64, name="qty", nullable=True)]) + + assert table.row_count == 2 + assert table.validity[0][0] & 0b11 == 0b10 + assert int(table.columns[0][1]) == 5 + + +@pytest.mark.parametrize("value", [(1, ColumnType.I64), (None, 1), (None,), [None, ColumnType.I64]]) +def test_write_row_rejects_a_malformed_typed_blank(tmp_path, value): + with open_writer(tmp_path / "bad-blank.xlsx") as writer: + writer.start_sheet("Data") + with pytest.raises(TypeError): + writer.write_row(["fine", value]) + writer.end_sheet() + + def test_out_of_order_call_raises(tmp_path): from excelreader import ExcelReaderError path = tmp_path / "bad.xlsx" with open_writer(path) as writer: with pytest.raises(ExcelReaderError): - writer.write_str("too early") + writer.write_row(["too early"]) writer.start_sheet("Data") - writer.start_row() - writer.end_row() + writer.write_row([]) writer.end_sheet() @@ -95,9 +114,7 @@ def test_closing_twice_is_safe(tmp_path): path = tmp_path / "twice.xlsx" writer = open_writer(path) writer.start_sheet("Data") - writer.start_row() - writer.write_i64(1) - writer.end_row() + writer.write_row([1]) writer.end_sheet() writer.close() writer.close() @@ -142,80 +159,88 @@ def test_write_row_rejects_an_unsupported_type(tmp_path): assert "1" in str(excinfo.value), "the message names the position" -def test_write_row_matches_explicit_calls(tmp_path): - sugar = tmp_path / "sugar.xlsx" - explicit = tmp_path / "explicit.xlsx" - - with open_writer(sugar) as writer: +def test_an_aware_datetime_is_written_in_utc(tmp_path): + path = tmp_path / "aware.xlsx" + offset = datetime.timezone(datetime.timedelta(hours=-3)) + with open_writer(path) as writer: writer.start_sheet("Data") - writer.write_row(["a", 1, 2.5, False]) + writer.write_row(["at"]) + writer.write_row([datetime.datetime(2021, 6, 7, 5, 0, tzinfo=offset)]) + writer.write_row([datetime.datetime(2021, 6, 7, 8, 0)]) writer.end_sheet() - with open_writer(explicit) as writer: - writer.start_sheet("Data") - writer.start_row() - writer.write_str("a") - writer.write_i64(1) - writer.write_f64(2.5) - writer.write_bool(False) - writer.end_row() - writer.end_sheet() + with open_workbook(path) as workbook: + stamps = workbook.sheets[0].parse([ColumnSpec(ColumnType.TIMESTAMP, name="at")]).columns[0] - def read(path): - with open_workbook(path) as workbook: - return [[(cell.type, cell.value) for cell in row] for row in workbook.sheets[0].rows()] + assert stamps[0] == stamps[1] - assert read(sugar) == read(explicit) +def test_a_writer_over_a_bytesio_hands_it_the_workbook_on_close(): + import io -def test_in_memory_writer_round_trips(): - from excelreader import open_bytes, open_writer_to_memory + from excelreader import open_workbook - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: writer.start_sheet("Data") writer.write_row(["name", "qty"]) writer.write_row(["widget", 7]) writer.end_sheet() - payload = writer.bytes() - - assert isinstance(payload, bytes) - assert len(payload) > 0 + assert buffer.getvalue() == b"" - with open_bytes(payload, format="xlsx") as workbook: + assert not buffer.closed + with open_workbook(buffer) as workbook: rows = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] assert rows[0] == ["name", "qty"] assert rows[1][0] == "widget" -def test_bytes_on_a_file_writer_raises(tmp_path): - from excelreader import ExcelReaderError +def test_a_writer_over_a_bytesio_finishes_a_sheet_left_open(): + import io - path = tmp_path / "file.xlsx" - with open_writer(path) as writer: + from excelreader import open_workbook + + buffer = io.BytesIO() + with open_writer(buffer, "xlsb") as writer: writer.start_sheet("Data") writer.write_row(["a"]) - writer.end_sheet() - with pytest.raises(ExcelReaderError): - writer.bytes() + with open_workbook(buffer) as workbook: + assert [row[0].value for row in workbook.sheets[0].rows()] == ["a"] + + +def test_a_writer_over_a_stream_needs_a_format_and_a_write_method(): + import io -def test_write_workbook_to_bytes_matches_write_workbook(tmp_path, xlsx_path): - from excelreader import open_bytes, write_workbook, write_workbook_to_bytes + with pytest.raises(ValueError, match="format"): + open_writer(io.BytesIO()) + with pytest.raises(ValueError, match="unknown format"): + open_writer(io.BytesIO(), "parquet") + with pytest.raises(TypeError, match="destination"): + open_writer(42, "xlsx") + + +def test_write_workbook_to_a_bytesio_matches_write_workbook_to_a_file(tmp_path, xlsx_path): + import io + + from excelreader import open_workbook, write_workbook with open_workbook(xlsx_path) as workbook: schema = workbook.sheets[0].infer_schema() - table = workbook.sheets[0].parse_typed(schema) + table = workbook.sheets[0].parse(schema) types = [spec.type for spec in schema] path = tmp_path / "from-file.xlsx" write_workbook(path, table, types, format="xlsx") - payload = write_workbook_to_bytes(table, types, format="xlsx") + buffer = io.BytesIO() + write_workbook(buffer, table, types, format="xlsx") + payload = buffer.getvalue() with open_workbook(path) as workbook: from_file = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] - with open_bytes(payload, format="xlsx") as workbook: + with open_workbook(payload, format="xlsx") as workbook: from_memory = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] assert from_file == from_memory @@ -227,10 +252,5 @@ def test_an_integer_beyond_64_bits_is_refused_not_wrapped(tmp_path, value): writer.start_sheet("Data") with pytest.raises(OverflowError): writer.write_row(["fine", value]) - writer.start_row() - with pytest.raises(OverflowError): - writer.write_i64(value) - writer.write_i64(2**63 - 1) - writer.write_i64(-(2**63)) - writer.end_row() + writer.write_row([2**63 - 1, -(2**63)]) writer.end_sheet() diff --git a/rust/excelreader-derive/src/lib.rs b/rust/excelreader-derive/src/lib.rs index 5ef608ca..4e81c01e 100644 --- a/rust/excelreader-derive/src/lib.rs +++ b/rust/excelreader-derive/src/lib.rs @@ -1,5 +1,5 @@ //! `#[derive(ExcelMapper)]` for excelreader's `ExcelMapper` trait. Do not depend on this crate -//! directly - `excelreader` re-exports the macro from `excelreader::workbook::ExcelMapper` +//! directly - `excelreader` re-exports the macro as `excelreader::ExcelMapper` //! (same identifier as the trait; derive macros and traits occupy different namespaces, so //! there's no collision, same as `serde`/`serde_derive`). @@ -41,23 +41,23 @@ fn expand(input: DeriveInput) -> syn::Result { } Ok(quote! { - impl ::excelreader::workbook::ExcelMapper for #struct_name { - fn bindings() -> ::std::vec::Vec<::excelreader::workbook::ColumnBinding> { + impl ::excelreader::ExcelMapper for #struct_name { + fn bindings() -> ::std::vec::Vec<::excelreader::__private::ColumnBinding> { ::std::vec![ #(#bindings),* ] } } - impl ::excelreader::writer::ExcelWriter for #struct_name { + impl ::excelreader::ExcelWriter for #struct_name { fn to_columns( rows: &[Self], ) -> ::core::result::Result< - ::std::vec::Vec<::excelreader::writer::OwnedColumn>, + ::std::vec::Vec<::excelreader::__private::OwnedColumn>, ::excelreader::Error, > { let n = rows.len(); #(#declarations)* for (row, r) in rows.iter().enumerate() { - let _ = row; + let _ = row; #(#pushes)* } ::core::result::Result::Ok(::std::vec![ #(#columns),* ]) @@ -109,7 +109,7 @@ fn parse_field(field: &Field) -> syn::Result> { fn field_binding(parsed: &ParsedField<'_>) -> proc_macro2::TokenStream { let field_ident = parsed.ident; let name = &parsed.names; - let xl_type = parsed.kind.xl_type_tokens(); + let column_type = parsed.kind.column_type_tokens(); let value = parsed.kind.value_tokens(); let assign_value = if parsed.is_option { quote! { ::std::option::Option::Some(#value) } @@ -118,9 +118,9 @@ fn field_binding(parsed: &ParsedField<'_>) -> proc_macro2::TokenStream { }; quote! { - ::excelreader::workbook::ColumnBinding { + ::excelreader::__private::ColumnBinding { names: &[#(#name),*], - xl_type: #xl_type, + column_type: #column_type, assign: |r, col, row| r.#field_ident = #assign_value, } } @@ -262,7 +262,7 @@ impl WriteField { quote! { match &r.#ident { ::core::option::Option::Some(value) => { - ::excelreader::writer::set_valid(&mut #validity, row); + ::excelreader::__private::set_valid(&mut #validity, row); #append } ::core::option::Option::None => { #placeholder } @@ -277,12 +277,11 @@ impl WriteField { let field = self.name.value(); match self.kind { FieldKind::Str => quote! { - ::excelreader::writer::push_str(&mut #offsets, &mut #data, value.as_str())?; + ::excelreader::__private::push_str(&mut #offsets, &mut #data, value.as_str())?; }, FieldKind::Int => quote! { #values.push( - i64::try_from(*value).map_err(|_| ::excelreader::Error::from_status( - ::excelreader::XL_INVALID_ARGUMENT, + i64::try_from(*value).map_err(|_| ::excelreader::__private::invalid_argument( ::std::format!( "field `{}` holds a value that does not fit an i64 column", #field @@ -315,7 +314,7 @@ impl WriteField { fn placeholder_tokens(&self) -> proc_macro2::TokenStream { let (values, offsets, data) = (&self.values, &self.offsets, &self.data); if matches!(self.kind, FieldKind::Str) { - return quote! { ::excelreader::writer::push_str(&mut #offsets, &mut #data, "")?; }; + return quote! { ::excelreader::__private::push_str(&mut #offsets, &mut #data, "")?; }; } quote! { #values.push(::core::default::Default::default()); } } @@ -326,15 +325,15 @@ impl WriteField { let name = &self.name; let payload = match self.kind { FieldKind::Str => quote! { - ::excelreader::writer::OwnedColumnData::Str { offsets: #offsets, data: #data } + ::excelreader::__private::OwnedColumnData::Str { offsets: #offsets, data: #data } }, - FieldKind::Int => quote! { ::excelreader::writer::OwnedColumnData::I64(#values) }, - FieldKind::Float => quote! { ::excelreader::writer::OwnedColumnData::F64(#values) }, - FieldKind::Bool => quote! { ::excelreader::writer::OwnedColumnData::Bool(#values) }, - FieldKind::Date => quote! { ::excelreader::writer::OwnedColumnData::Date(#values) }, - FieldKind::Time => quote! { ::excelreader::writer::OwnedColumnData::Time(#values) }, + FieldKind::Int => quote! { ::excelreader::__private::OwnedColumnData::I64(#values) }, + FieldKind::Float => quote! { ::excelreader::__private::OwnedColumnData::F64(#values) }, + FieldKind::Bool => quote! { ::excelreader::__private::OwnedColumnData::Bool(#values) }, + FieldKind::Date => quote! { ::excelreader::__private::OwnedColumnData::Date(#values) }, + FieldKind::Time => quote! { ::excelreader::__private::OwnedColumnData::Time(#values) }, FieldKind::Timestamp => { - quote! { ::excelreader::writer::OwnedColumnData::Timestamp(#values) } + quote! { ::excelreader::__private::OwnedColumnData::Timestamp(#values) } } }; let validity_expr = if self.is_option { @@ -343,7 +342,7 @@ impl WriteField { quote! { ::core::option::Option::None } }; quote! { - ::excelreader::writer::OwnedColumn { + ::excelreader::__private::OwnedColumn { name: ::core::option::Option::Some(#name), data: #payload, validity: #validity_expr, @@ -387,35 +386,35 @@ impl FieldKind { } } - fn xl_type_tokens(&self) -> proc_macro2::TokenStream { + fn column_type_tokens(&self) -> proc_macro2::TokenStream { match self { - FieldKind::Str => quote! { ::excelreader::XL_T_STRING }, - FieldKind::Int => quote! { ::excelreader::XL_T_I64 }, - FieldKind::Float => quote! { ::excelreader::XL_T_F64 }, - FieldKind::Bool => quote! { ::excelreader::XL_T_BOOL }, - FieldKind::Date => quote! { ::excelreader::XL_T_DATE }, - FieldKind::Time => quote! { ::excelreader::XL_T_TIME }, - FieldKind::Timestamp => quote! { ::excelreader::XL_T_TIMESTAMP }, + FieldKind::Str => quote! { ::excelreader::ColumnType::String }, + FieldKind::Int => quote! { ::excelreader::ColumnType::I64 }, + FieldKind::Float => quote! { ::excelreader::ColumnType::F64 }, + FieldKind::Bool => quote! { ::excelreader::ColumnType::Bool }, + FieldKind::Date => quote! { ::excelreader::ColumnType::Date }, + FieldKind::Time => quote! { ::excelreader::ColumnType::Time }, + FieldKind::Timestamp => quote! { ::excelreader::ColumnType::Timestamp }, } } fn value_tokens(&self) -> proc_macro2::TokenStream { match self { - FieldKind::Str => quote! { ::excelreader::workbook::column_str(col, row).to_string() }, + FieldKind::Str => quote! { ::excelreader::__private::column_str(col, row).to_string() }, FieldKind::Int => quote! { - ::core::convert::TryFrom::try_from(::excelreader::workbook::column_i64(col, row)) + ::core::convert::TryFrom::try_from(::excelreader::__private::column_i64(col, row)) .expect("value does not fit this field's integer type") }, - FieldKind::Float => quote! { ::excelreader::workbook::column_f64(col, row) as _ }, - FieldKind::Bool => quote! { ::excelreader::workbook::column_bool(col, row) }, + FieldKind::Float => quote! { ::excelreader::__private::column_f64(col, row) as _ }, + FieldKind::Bool => quote! { ::excelreader::__private::column_bool(col, row) }, FieldKind::Date => { - quote! { ::core::convert::Into::into(::excelreader::workbook::column_date(col, row)) } + quote! { ::core::convert::Into::into(::excelreader::__private::column_date(col, row)) } } FieldKind::Time => { - quote! { ::core::convert::Into::into(::excelreader::workbook::column_time(col, row)) } + quote! { ::core::convert::Into::into(::excelreader::__private::column_time(col, row)) } } FieldKind::Timestamp => { - quote! { ::core::convert::Into::into(::excelreader::workbook::column_timestamp(col, row)) } + quote! { ::core::convert::Into::into(::excelreader::__private::column_timestamp(col, row)) } } } } @@ -459,11 +458,11 @@ mod tests { ) .expect("expand must succeed"); - assert!(output.contains("impl :: excelreader :: workbook :: ExcelMapper for Row")); - assert!(output.contains("XL_T_STRING")); - assert!(output.contains("XL_T_I64")); - assert!(output.contains("XL_T_F64")); - assert!(output.contains("XL_T_BOOL")); + assert!(output.contains("impl :: excelreader :: ExcelMapper for Row")); + assert!(output.contains("ColumnType :: String")); + assert!(output.contains("ColumnType :: I64")); + assert!(output.contains("ColumnType :: F64")); + assert!(output.contains("ColumnType :: Bool")); assert!(output.contains("\"Nome\"")); assert!(output.contains("\"Idade\"")); assert!(output.contains("\"Peso\"")); @@ -483,7 +482,7 @@ mod tests { .expect("expand must succeed"); assert!(output.contains("Option :: Some")); - assert!(output.contains("XL_T_BOOL")); + assert!(output.contains("ColumnType :: Bool")); } #[test] @@ -525,7 +524,10 @@ mod tests { )) .unwrap_or_else(|e| panic!("{ty} must be supported: {e}")); - assert!(output.contains("XL_T_I64"), "{ty} must use XL_T_I64"); + assert!( + output.contains("ColumnType :: I64"), + "{ty} must use an I64 column" + ); assert!(output.contains("try_from"), "{ty} must convert via TryFrom"); } } @@ -537,19 +539,22 @@ mod tests { r#"struct Row {{ #[excel(name = "V")] v: {ty}, }}"# )) .unwrap_or_else(|e| panic!("{ty} must be supported: {e}")); - assert!(output.contains("XL_T_F64"), "{ty} must use XL_T_F64"); + assert!( + output.contains("ColumnType :: F64"), + "{ty} must use an F64 column" + ); } } #[test] fn maps_temporal_types_onto_their_own_column_types() { for (ty, expected) in [ - ("Date", "XL_T_DATE"), - ("NaiveDate", "XL_T_DATE"), - ("Time", "XL_T_TIME"), - ("NaiveTime", "XL_T_TIME"), - ("Timestamp", "XL_T_TIMESTAMP"), - ("NaiveDateTime", "XL_T_TIMESTAMP"), + ("Date", "ColumnType :: Date"), + ("NaiveDate", "ColumnType :: Date"), + ("Time", "ColumnType :: Time"), + ("NaiveTime", "ColumnType :: Time"), + ("Timestamp", "ColumnType :: Timestamp"), + ("NaiveDateTime", "ColumnType :: Timestamp"), ] { let output = expand_str(&format!( r#"struct Row {{ #[excel(name = "D")] d: {ty}, }}"# @@ -571,7 +576,7 @@ mod tests { ) .expect("a qualified path must resolve by its last segment"); - assert!(output.contains("XL_T_DATE")); + assert!(output.contains("ColumnType :: Date")); } #[test] @@ -621,7 +626,7 @@ mod tests { ) .expect("expand must succeed"); - assert!(output.contains("impl :: excelreader :: writer :: ExcelWriter for Row")); + assert!(output.contains("impl :: excelreader :: ExcelWriter for Row")); assert!( output.contains("push_str"), "a String field must go through push_str" diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 59549d99..3be52cfe 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -2,19 +2,20 @@ # excelreader (Rust) -Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or -memory, with the full open-options surface), sheet navigation, schema inference, schema-driven typed -parse, schema-driven writing, row-by-row decoded reads, and Arrow export. +Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path, +memory, your own random-access source or any `Read`), sheet navigation, schema inference, +schema-driven typed parse, schema-driven writing, row-by-row reads, parallel CSV aggregation, and +Arrow export. Every public item lives at the crate root. ## Usage ```toml [dependencies] -excelreader = "2.1" +excelreader = "7.0" ``` ```rust,no_run -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::{ExcelMapper, Workbook, ReadOptions}; #[derive(Default, ExcelMapper)] struct Row { @@ -25,24 +26,27 @@ struct Row { } let workbook = Workbook::open("book.xlsx")?; -let table = parse_sheet::(workbook.sheet(0)?, 1)?; +let table = workbook.sheet(0)?.parse::(&ReadOptions::new())?; for row in table.iter() { /* ... */ } # Ok::<(), excelreader::Error>(()) ``` -`parse_sheet` takes a `Sheet`, which `Workbook::sheet` hands out (see below). +`Sheet::parse(header_row, degree_of_parallelism)` converts on the native side and never formats a +cell to text, so it is the fastest way to read a sheet whose columns you know. `header_row` is +1-based (`0` = no header); `degree_of_parallelism` is `0` for every processor, `1` sequential, `n` +up to `n` threads - XLSX, XLSB and CSV are split, and the table is the same whatever it is. ### Field types | Field type | Column | |---|---| -| `String` | `XL_T_STRING` | -| `i8`..`i64`, `isize`, `u8`..`u64`, `usize` | `XL_T_I64` | -| `f32`, `f64` | `XL_T_F64` | -| `bool` | `XL_T_BOOL` | -| `Date` / `chrono::NaiveDate` | `XL_T_DATE` | -| `Time` / `chrono::NaiveTime` | `XL_T_TIME` | -| `Timestamp` / `chrono::NaiveDateTime` | `XL_T_TIMESTAMP` | +| `String` | `ColumnType::String` | +| `i8`..`i64`, `isize`, `u8`..`u64`, `usize` | `ColumnType::I64` | +| `f32`, `f64` | `ColumnType::F64` | +| `bool` | `ColumnType::Bool` | +| `Date` / `chrono::NaiveDate` | `ColumnType::Date` | +| `Time` / `chrono::NaiveTime` | `ColumnType::Time` | +| `Timestamp` / `chrono::NaiveDateTime` | `ColumnType::Timestamp` | `Option` of any of the above makes the field `None` when the cell is null; a non-`Option` field is left at its `Default` instead. @@ -52,114 +56,108 @@ wrapping silently. `Date`/`Time`/`Timestamp` are dependency-free newtypes over t representation; enable the `chrono` feature to use `chrono`'s calendar types directly: ```toml -excelreader = { version = "2.1", features = ["chrono"] } +excelreader = { version = "7.0", features = ["chrono"] } ``` -### Sheets and schema inference +### Opening + +`Workbook::open(path)` sniffs the format and uses every library default. Everything else goes +through `OpenOptions`, in the style of `std::fs::OpenOptions`: set what you need, then call `open`, +`open_bytes`, `open_source` or `open_reader`. ```rust,no_run -# use excelreader::workbook::Workbook; -let workbook = Workbook::open("book.xlsx")?; -for (index, name) in workbook.sheet_names()?.iter().enumerate() { - println!("{index}: {name}"); -} +use excelreader::{Format, OpenOptions}; -// Guess a schema from the header row plus a sample of the data, before committing to one. -for column in workbook.sheet(1)?.infer_schema(1, 100)? { - println!("{:?} -> type {}", column.name, column.column_type); -} +let options = OpenOptions::new().format(Format::Csv).csv_delimiter(b';' as i32); +let from_path = options.open("data.csv")?; +let from_memory = options.open_bytes(b"a;b\n1;2\n")?; # Ok::<(), excelreader::Error>(()) ``` -`infer_schema_parse_text(header_row, sample_size)` also types text cells (every CSV field, or -numbers stored as text): integers, decimals, `true`/`false` and ISO dates. `parse_sheet_parallel` -and `arrow::parse_arrow_parallel` take a `degree_of_parallelism` (`0` = every processor, `1` = -sequential); XLSX, XLSB and CSV are split, and the result equals `parse_sheet`'s. +Format sniffing (`Format::Auto`, the default) does not detect CSV - set `Format::Csv`. The +resource limits (`max_total_decompressed_bytes`, `max_cell_bytes`, `max_shared_string_bytes`, +`max_zip_entries`, `max_buffered_bytes`) protect against hostile files; lower them when opening +untrusted input. + +### Sheets and schema inference A workbook hands out sheets. A `Sheet` is the workbook and an index: it holds no native resource, it is `Copy`, and every read on it opens its own cursor. ```rust,no_run -# use excelreader::workbook::Workbook; +# use excelreader::Workbook; let workbook = Workbook::open("report.xlsx")?; for sheet in workbook.sheets()? { println!("{} {:?}", sheet.name()?, sheet.visibility()?); } +// Guess a schema from the header row plus a sample of the data, before committing to one. let sheet = workbook.sheet_by_name("Totals")?.expect("the workbook has a Totals sheet"); -let mut rows = sheet.rows()?; -while let Some(row) = rows.next_row() { - let row = row?; - // ... +for column in sheet.infer_schema(1, 100, false)? { + println!("{:?} -> {:?}", column.name, column.column_type); } # Ok::<(), excelreader::Error>(()) ``` +`infer_schema`'s last argument, `parse_text`, also types text cells (every CSV field, or numbers +stored as text): integers, decimals, `true`/`false` and ISO dates. + `Workbook` is `Send + Sync`, so sheets can be read in parallel over one open workbook, which loads the shared-string table once: ```rust,no_run -# use excelreader::workbook::Workbook; +# use excelreader::Workbook; # let workbook = Workbook::open("report.xlsx")?; let tables = std::thread::scope(|scope| { let handles: Vec<_> = workbook .sheets()? .into_iter() - .map(|sheet| scope.spawn(move || sheet.read_all_blob())) + .map(|sheet| scope.spawn(move || sheet.read_all())) .collect(); handles.into_iter().map(|handle| handle.join().expect("no panic")).collect::, _>>() })?; # Ok::<(), excelreader::Error>(()) ``` -`Workbook::open_with` takes an explicit format and `OpenOptions`; `Workbook::open_memory` reads from -a byte slice. Note that format sniffing does not detect CSV - pass `XL_FORMAT_CSV` explicitly. - ### Encrypted workbooks -`OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm) through -`Workbook::open_with`/`open_memory`: +`OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm): ```rust,no_run -use excelreader::workbook::Workbook; -use excelreader::{Error, OpenOptions, XL_FORMAT_AUTO}; +use excelreader::{ErrorKind, OpenOptions}; -let options = OpenOptions::new().password("hunter2"); -match Workbook::open_with("protected.xlsx", XL_FORMAT_AUTO, Some(&options)) { +match OpenOptions::new().password("hunter2").open("protected.xlsx") { Ok(workbook) => { /* ... */ } - Err(Error::PasswordIncorrect { .. }) => { /* ask again */ } - Err(Error::PasswordRequired { .. }) => { /* no password was supplied */ } - Err(other) => { /* Error::Native - unsupported scheme or a corrupt file; not worth retrying */ } + Err(e) if e.kind() == ErrorKind::PasswordIncorrect => { /* ask again */ } + Err(e) if e.kind() == ErrorKind::PasswordRequired => { /* no password was supplied */ } + Err(other) => { /* unsupported scheme or a corrupt file; not worth retrying */ } } ``` -`Error::PasswordRequired`/`Error::PasswordIncorrect` are dedicated variants rather than the generic -error carrying `XL_STATUS_PASSWORD_REQUIRED`/`XL_STATUS_PASSWORD_INCORRECT`, so callers can -`match`/`matches!` on "needs a password" without hardcoding the status codes. Format sniffing does not -see through the CFB wrapper an encrypted file is stored in - pass `XL_FORMAT_AUTO` explicitly (as -above) rather than `XL_FORMAT_XLSX`, or the file is read as a plain ZIP and fails before the password -is ever checked. +Every `Error` carries an `ErrorKind` to branch on and the native detail in `message()`. Format +sniffing does not see through the CFB wrapper an encrypted file is stored in, so keep the default +`Format::Auto` rather than setting `Format::Xlsx`, or the file is read as a plain ZIP and fails +before the password is ever checked. Writing an encrypted workbook is a second step: write the plaintext package with `write_columns`/ `write_sheet`, then wrap it with `encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container - AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: ```rust,no_run -use excelreader::writer::{encrypt_package, write_columns}; -use excelreader::XL_FORMAT_XLSX; -# use excelreader::writer::{Column, ColumnData}; +use excelreader::{encrypt_package, write_columns, WriteOptions}; +# use excelreader::{Column, ColumnData}; # let ids = [1i64, 2, 3]; -# let columns = [Column { name: Some("id"), data: ColumnData::I64(&ids), validity: None }]; +# let columns = [Column::new(Some("id"), ColumnData::I64(&ids))]; -write_columns("plain.xlsx", XL_FORMAT_XLSX, &columns, None)?; +write_columns("plain.xlsx", &columns, &WriteOptions::new())?; encrypt_package("plain.xlsx", "secret.xlsx", "hunter2")?; # Ok::<(), excelreader::Error>(()) ``` `package_path` is read twice (it is not disposed or removed), so it must already be a finished file. Encryption parameters are fixed at Excel's own defaults - there are no options - and only XLSX/XLSB -packages can be encrypted, matching what `Workbook::open_with`/`open_memory` can decrypt. +packages can be encrypted, matching what `OpenOptions::password` can decrypt. ### Reading from your own bytes @@ -185,15 +183,15 @@ impl excelreader::Source for HttpSource { } } -let workbook = Workbook::open_source(source, XL_FORMAT_AUTO, None)?; +let workbook = OpenOptions::new().open_source(source)?; ``` Anything that implements `Read` works through `open_reader`. A CSV is read as it arrives; an XLSX, XLSB or XLS is read whole first. ```rust,no_run -# use excelreader::{workbook::Workbook, XL_FORMAT_CSV}; -let workbook = Workbook::open_reader(std::io::stdin(), XL_FORMAT_CSV, None)?; +# use excelreader::{Format, OpenOptions}; +let workbook = OpenOptions::new().format(Format::Csv).open_reader(std::io::stdin())?; # Ok::<(), excelreader::Error>(()) ``` @@ -203,19 +201,18 @@ needed the bytes. ### Arrow export (`arrow` feature) ```toml -excelreader = { version = "2.1", features = ["arrow"] } +excelreader = { version = "7.0", features = ["arrow"] } ``` ```rust,no_run # #[cfg(feature = "arrow")] # fn main() -> Result<(), excelreader::Error> { -use excelreader::arrow::parse_arrow; -use excelreader::workbook::Workbook; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::{Workbook, ReadOptions}; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; -let batch = parse_arrow::(workbook.sheet(0)?, 1)?; +let batch = workbook.sheet(0)?.to_arrow::(&ReadOptions::new())?; println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); # Ok(()) # } @@ -225,22 +222,21 @@ println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); Off by default — arrow-rs is a large dependency and the typed-parse path needs none of it. -Batched: `parse_arrow_stream` is `parse_arrow` delivered a batch at a time, returning -`ArrowChunks<'_>`, an `Iterator>` that also implements +Batched: `Sheet::to_arrow_stream` is `parse_arrow` delivered a batch at a time, returning +`ArrowBatches<'_>`, an `Iterator>` that also implements `RecordBatchReader`: ```rust,no_run # #[cfg(feature = "arrow")] # fn main() -> Result<(), excelreader::Error> { -use excelreader::arrow::parse_arrow_stream; -use excelreader::workbook::Workbook; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::{Workbook, ReadOptions}; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; -let mut chunks = parse_arrow_stream::(workbook.sheet(0)?, 1, 10_000)?; +let mut chunks = workbook.sheet(0)?.to_arrow_stream::(&ReadOptions::new())?; while let Some(batch) = chunks.next() { - let Ok(batch) = batch else { break }; // ArrowChunks does not fuse, unlike TypedChunks + let Ok(batch) = batch else { break }; // ArrowBatches does not fuse, unlike TypedBatches println!("{} rows", batch.num_rows()); } # Ok(()) @@ -249,8 +245,8 @@ while let Some(batch) = chunks.next() { # fn main() {} ``` -Same `batch_size` semantics as `typed_chunks` below. Unlike `TypedChunks`, -`ArrowChunks` does not fuse after an error — it follows arrow-rs's own semantics — so break on the +Same `batch_size` semantics as `parse_batches` below. Unlike `TypedBatches`, +`ArrowBatches` does not fuse after an error — it follows arrow-rs's own semantics — so break on the first `Err` rather than continuing the loop. ## Writing @@ -259,13 +255,12 @@ first `Err` rather than continuing the loop. types in the table above apply unchanged: ```rust,no_run -use excelreader::writer::write_sheet; -use excelreader::XL_FORMAT_XLSX; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::{write_sheet, WriteOptions}; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } # let rows = vec![Row::default()]; -write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +write_sheet("out.xlsx", &rows, &WriteOptions::new())?; # Ok::<(), excelreader::Error>(()) ``` @@ -278,16 +273,11 @@ lifetimes on `Column<'a>` are what turn the ABI's borrow contract into something checks: ```rust,no_run -use excelreader::writer::{write_columns, Column, ColumnData}; -use excelreader::XL_FORMAT_XLSX; +use excelreader::{write_columns, Column, ColumnData, WriteOptions}; let ids = [1i64, 2, 3]; -let columns = [Column { - name: Some("id"), - data: ColumnData::I64(&ids), - validity: None, -}]; -write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; +let columns = [Column::new(Some("id"), ColumnData::I64(&ids))]; +write_columns("out.xlsx", &columns, &WriteOptions::new())?; # Ok::<(), excelreader::Error>(()) ``` @@ -295,39 +285,53 @@ write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; length and reads `(rows + 7) / 8` bytes on trust, so a short slice would be a buffer overrun rather than a wrong answer. -`WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. -`format_from_path` infers the format from an extension; it returns `XL_FORMAT_AUTO` for anything it -does not recognize, which the write then rejects - a file being created has no signature bytes to -sniff, so there is nothing to fall back on. +`WriteOptions` sets the format, the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB +toggles. Its format defaults to `Format::Auto`, inferred from the path's extension; an extension it +does not recognize is an error - a file being created has no signature bytes to sniff. + +Each write has a path form and an `io::Write` form: `write_columns_to`/`write_sheet_to` build the +workbook in memory and write it to the writer in one `write_all`. A writer has no extension, so +these need an explicit format: + +```rust,no_run +use excelreader::{write_sheet_to, Format, WriteOptions}; +# #[derive(Default, excelreader::ExcelMapper)] +# struct Row { #[excel(name = "Name")] name: String } +# let rows = vec![Row::default()]; + +let mut bytes = Vec::new(); +write_sheet_to(&mut bytes, &rows, &WriteOptions::new().format(Format::Xlsx))?; +# Ok::<(), excelreader::Error>(()) +``` `write_sheet` walks the slice once and appends each field to its own buffer, monomorphized per field. That transpose is the only copy it makes; `write_columns` pays nothing. ### Streaming writes -`write_sheet`/`write_columns` build the whole table in memory first. `writer_handle::WriterHandle` is -the row-by-row alternative, writing directly as each call arrives instead: +`write_sheet`/`write_columns` build the whole table in memory first. `WriterHandle` is the +row-by-row alternative, writing directly as each call arrives instead: ```rust,no_run -use excelreader::writer_handle::WriterHandle; +use excelreader::{WriteOptions, WriterHandle}; -let mut handle = WriterHandle::open("out.xlsx", None)?; +let mut handle = WriterHandle::create("out.xlsx", &WriteOptions::new())?; handle.start_sheet("Summary")?; handle.start_row()?; handle.write_str(Some("Name"))?; handle.write_i64(Some(42))?; handle.end_row()?; handle.end_sheet()?; +handle.finish()?; # Ok::<(), excelreader::Error>(()) ``` -Call order mirrors the C ABI's `xl_writer_handle`: `open`/`open_with`/`open_memory`, then per sheet -`start_sheet..end_sheet`, each containing `start_row..end_row` with one `write_*` call per cell -(`None` writes a blank cell), left to right. A call out of order returns `Err` rather than -corrupting output. `open_memory` backs the handle with an in-memory buffer instead of a file; read -it out with `bytes()`. Dropping a `WriterHandle` closes and releases it, same as `Workbook` — call -`bytes()` (memory-backed) or reopen the path (file-backed) to observe the result rather than relying -on the drop for that. +Call order mirrors the C ABI's `xl_writer_handle`: `create` (a path) or `new` (any `io::Write`), +then per sheet `start_sheet..end_sheet`, each containing `start_row..end_row` with one `write_*` +call per cell (`None` writes a blank cell), left to right, then `finish`. A call out of order +returns `Err` rather than corrupting output. A handle from `new` writes the whole workbook to its +writer once, in `finish`. Dropping a handle without finishing it closes it on a best-effort basis, +discarding any error, and writes nothing to a writer. ## Reading rows one at a time @@ -335,7 +339,7 @@ on the drop for that. reuses, so iterating a sheet allocates nothing per row: ```rust,no_run -use excelreader::workbook::Workbook; +use excelreader::Workbook; let workbook = Workbook::open("book.xlsx")?; let mut cursor = workbook.sheet(0)?.rows()?; @@ -350,34 +354,31 @@ while let Some(row) = cursor.next_row() { ``` Because each row borrows the cursor's buffer, only one row is alive at a time — the borrow checker -enforces it. To hold every row at once, use `read_all_decoded`, which decodes the whole -sheet in one native call and keeps it alive until dropped: +enforces it. To hold every row at once, use `read_all`, which reads the whole sheet in one native +call into one buffer and keeps it alive until dropped: ```rust,no_run -# use excelreader::workbook::Workbook; +# use excelreader::Workbook; let workbook = Workbook::open("book.xlsx")?; -let rows = workbook.sheet(0)?.read_all_decoded()?; +let rows = workbook.sheet(0)?.read_all()?; for row in rows.iter() { println!("{} cells", row.len()); } # Ok::<(), excelreader::Error>(()) ``` -`parse_sheet` remains the fastest way to read a sheet whose columns you know — it converts on the -native side and never formats a cell to text. - ### Chunked reads -For a sheet too large to hold in memory at once, `Sheet::typed_chunks` is `parse_sheet` delivered +For a sheet too large to hold in memory at once, `Sheet::parse_batches` is `Sheet::parse` delivered a batch at a time: ```rust,no_run -use excelreader::workbook::Workbook; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::{Workbook, ReadOptions}; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; -let chunks = workbook.sheet(0)?.typed_chunks::(1, 10_000)?; +let chunks = workbook.sheet(0)?.parse_batches::(&ReadOptions::new())?; for batch in chunks { let batch = batch?; for row in batch.iter() { /* ... */ } @@ -385,34 +386,64 @@ for batch in chunks { # Ok::<(), excelreader::Error>(()) ``` -`typed_chunks` returns `TypedChunks<'_, Row>`, an `Iterator, Error>>` +`parse_batches` returns `TypedBatches<'_, Row>`, an `Iterator, Error>>` that closes the native reader on drop. `batch_size` is rows per batch — `0` means one unbounded -batch, identical to `parse_sheet`; negative is an error. Chunked reads, of either kind (typed or +batch, identical to `parse`; negative is an error. Chunked reads, of either kind (typed or Arrow — see [Arrow export](#arrow-export-arrow-feature) above), are independent: several can be live -on one workbook, and other reads do not disturb them. `TypedChunks` ends after yielding an `Err`, so +on one workbook, and other reads do not disturb them. `TypedBatches` ends after yielding an `Err`, so a `for` loop stops there rather than spinning on it. +## Parallel CSV aggregation + +`aggregate_csv` folds a CSV file or buffer into one accumulator across several threads, without +building a table. The CSV dialect and cell limit come from `OpenOptions`: + +```rust,no_run +use excelreader::{aggregate_csv, CsvAccumulator, OpenOptions, RowRef}; + +struct Count(u64); +impl CsvAccumulator for Count { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + self.0 += 1; + Ok(()) + } + fn combine(&mut self, other: &mut Self) -> Result<(), i32> { + self.0 += other.0; + Ok(()) + } +} + +let options = OpenOptions::new().csv_delimiter(b';' as i32); +let rows = aggregate_csv("big.csv", &options, 1, 0, || Count(0))?; +# Ok::<(), excelreader::Error>(()) +``` + +The input is a path (`&str`, `&Path`, `&PathBuf`) or bytes (`&[u8]`); then `header_row` and +`degree_of_parallelism`, as in `Sheet::parse`. A nonzero code from a callback comes back as +`ErrorKind::Callback(code)`; a panic in one is resumed on the calling thread. + ## Bounds and panics -`TableView::get` returns `Option` and is `None` outside `0..len()`. The `column_*` accessors used -by generated bindings panic on an out-of-range row, a column type that does not match the binding, or -a string the native library returned as non-UTF-8 - each is a contract violation rather than -recoverable input. +`TableView::get` returns `Option` and is `None` outside `0..len()`. The accessors the derive's +generated code reads columns through panic on an out-of-range row, a column type that does not +match the binding, or a string the native library returned as non-UTF-8 - each is a contract +violation rather than recoverable input. ## Parallel aggregation on macOS -`aggregate_csv_file`/`aggregate_csv_memory` over a source large enough to partition (256 KB and up) -can crash the process on macOS arm64. The fault is inside the native runtime's signal handling, not -in this crate: a .NET thread resumes at address 0 after a signal the runtime sent it, which ends the -process with no message. It happens with any `degree_of_parallelism` above 1, and the same runtime -built into a plain C host does not show it, so a Rust host appears to make it far more likely. +`aggregate_csv` over a source large enough to partition (256 KB and up) can crash the process on +macOS arm64. The fault is inside the native runtime's signal handling, not in this crate: a .NET +thread resumes at address 0 after a signal the runtime sent it, which ends the process with no +message. It happens with any `degree_of_parallelism` above 1, and the same runtime built into a +plain C host does not show it, so a Rust host appears to make it far more likely. -`parse_sheet_parallel` and `arrow::parse_arrow_parallel` now fan out on XLSX and XLSB too; this has not been tested -on macOS, so treat them the same way. +`Sheet::parse` and `Sheet::to_arrow` with a `degree_of_parallelism` above 1 fan out on XLSX and +XLSB too; this has not been tested on macOS, so treat them the same way. -Until it is fixed upstream, pass `degree_of_parallelism: 1` on macOS for a sequential run, or keep -sources under the 256 KB partitioning floor, where the library never fans out. Linux and Windows are -unaffected. The crate's own partitioned aggregation tests are `#[ignore]`d on macOS for this reason. +Until it is fixed upstream, pass a `degree_of_parallelism` of 1 on macOS for a sequential run, or +keep sources under the 256 KB partitioning floor, where the library never fans out. Linux and +Windows are unaffected. The crate's own partitioned aggregation tests are `#[ignore]`d on macOS for +this reason. ## Build notes @@ -421,9 +452,9 @@ matching GitHub Release. Set `EXCELREADER_NATIVE_LIB_DIR` to a directory contain copy instead (e.g. from `dotnet publish ../../src/ExcelReader.Native -r win-x64`) to skip that download - useful when building from source before a release exists yet. -Every constructor first checks the loaded library's `xl_abi_version()` against the `XL_ABI_VERSION` -this crate was compiled against, and fails with a explanatory error rather than reading native memory -through a layout that may have changed. +Every constructor first checks the loaded library's ABI revision against the one this crate was +compiled against, and fails with an explanatory error rather than reading native memory through a +layout that may have changed. ## Benchmarks @@ -431,7 +462,7 @@ Criterion suite in `benches/`. Measured on Windows 10 (22H2), 16 logical CPUs @ rustc 1.97.1 (Release), Criterion 0.8, 100 samples per benchmark (medians shown). `write_bench` takes 20 samples instead — each of its iterations writes a whole 65,535-row file. -`benches/parse_bench.rs` - `open`/`parse_sheet`/`infer_schema`, same methodology as the C++ suite: +`benches/parse_bench.rs` - `open`/`Sheet::parse`/`infer_schema`, same methodology as the C++ suite: | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| @@ -451,7 +482,7 @@ not the file's total row count. into an owned value (`String` for text) and fold it into one accumulator, so neither side gets a zero-copy advantage the other can't take: -| Format | ExcelReader (`parse_sheet::`) | calamine (`worksheet_range` + `Data` match) | +| Format | ExcelReader (`Sheet::parse::`) | calamine (`worksheet_range` + `Data` match) | |---|---:|---:| | XLSX | 79.1 ms | 279.7 ms | | XLSB | 46.9 ms | 87.5 ms | diff --git a/rust/excelreader/benches/alloc_bench.rs b/rust/excelreader/benches/alloc_bench.rs index 67eb3d37..c7ca30b7 100644 --- a/rust/excelreader/benches/alloc_bench.rs +++ b/rust/excelreader/benches/alloc_bench.rs @@ -1,7 +1,8 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicU64, Ordering}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::ReadOptions; +use excelreader::{ExcelMapper, Workbook}; static BYTES_ALLOCATED: AtomicU64 = AtomicU64::new(0); @@ -93,7 +94,7 @@ fn main() { let small_path = fixture_path(); let large_path = large_fixture_path(); const ITERATIONS: usize = 200; - const ITERATIONS_LARGE: usize = 30; + const ITERATIONS_LARGE: usize = 30; println!("Bytes allocated per call, Rust binding (System allocator, this process only — excludes the NativeAOT side of the FFI boundary):\n"); @@ -112,7 +113,11 @@ fn main() { ITERATIONS, || Workbook::open(&small_path).expect("open must succeed"), |workbook| { - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("parse must succeed"); std::hint::black_box(table); }, ); @@ -122,7 +127,11 @@ fn main() { ITERATIONS, || Workbook::open(&small_path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 100).expect("infer_schema must succeed"); + let schema = workbook + .sheet(0) + .expect("sheet 0") + .infer_schema(1, 100, false) + .expect("infer_schema must succeed"); std::hint::black_box(schema); }, ); @@ -142,7 +151,11 @@ fn main() { ITERATIONS_LARGE, || Workbook::open(&large_path).expect("open must succeed"), |workbook| { - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("parse must succeed"); std::hint::black_box(table); }, ); @@ -152,7 +165,11 @@ fn main() { ITERATIONS_LARGE, || Workbook::open(&large_path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 1000).expect("infer_schema must succeed"); + let schema = workbook + .sheet(0) + .expect("sheet 0") + .infer_schema(1, 1000, false) + .expect("infer_schema must succeed"); std::hint::black_box(schema); }, ); diff --git a/rust/excelreader/benches/compare_bench.rs b/rust/excelreader/benches/compare_bench.rs index 5efb39e8..5695838b 100644 --- a/rust/excelreader/benches/compare_bench.rs +++ b/rust/excelreader/benches/compare_bench.rs @@ -1,8 +1,8 @@ - use calamine::{open_workbook_auto, Data, Reader}; use criterion::{criterion_group, criterion_main, Criterion}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; -use excelreader::Date; +use excelreader::ExcelMapper; +use excelreader::ReadOptions; +use excelreader::{Date, Format, OpenOptions}; #[derive(Default, ExcelMapper)] struct FullRow { @@ -79,13 +79,21 @@ fn accumulate_data_cell(acc: &mut i64, cell: &Data) { } } -fn bench_excelreader(c: &mut Criterion, group: &str, path: String, format: i32) { +fn bench_excelreader(c: &mut Criterion, group: &str, path: String, format: Format) { c.bench_function(group, |b| { b.iter_batched( - || Workbook::open_with(&path, format, None).expect("open must succeed"), + || { + OpenOptions::new() + .format(format) + .open(&path) + .expect("open must succeed") + }, |workbook| { - let table = - parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("parse must succeed"); let mut acc = 0i64; for row in table.iter() { acc += accumulate_full_row(&row); @@ -97,13 +105,21 @@ fn bench_excelreader(c: &mut Criterion, group: &str, path: String, format: i32) }); } -fn bench_excelreader_parse_only(c: &mut Criterion, group: &str, path: String, format: i32) { +fn bench_excelreader_parse_only(c: &mut Criterion, group: &str, path: String, format: Format) { c.bench_function(group, |b| { b.iter_batched( - || Workbook::open_with(&path, format, None).expect("open must succeed"), + || { + OpenOptions::new() + .format(format) + .open(&path) + .expect("open must succeed") + }, |workbook| { - let table = - parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("parse must succeed"); std::hint::black_box(table.len()); std::hint::black_box(table); }, @@ -132,7 +148,7 @@ fn bench_calamine(c: &mut Criterion, group: &str, path: String) { } fn bench_excelreader_xlsx(c: &mut Criterion) { - bench_excelreader(c, "excelreader_xlsx_full", xlsx_path(), excelreader::XL_FORMAT_XLSX); + bench_excelreader(c, "excelreader_xlsx_full", xlsx_path(), Format::Xlsx); } fn bench_calamine_xlsx(c: &mut Criterion) { @@ -140,7 +156,7 @@ fn bench_calamine_xlsx(c: &mut Criterion) { } fn bench_excelreader_xlsb(c: &mut Criterion) { - bench_excelreader(c, "excelreader_xlsb_full", xlsb_path(), excelreader::XL_FORMAT_XLSB); + bench_excelreader(c, "excelreader_xlsb_full", xlsb_path(), Format::Xlsb); } fn bench_calamine_xlsb(c: &mut Criterion) { @@ -148,21 +164,11 @@ fn bench_calamine_xlsb(c: &mut Criterion) { } fn bench_excelreader_xlsx_parse_only(c: &mut Criterion) { - bench_excelreader_parse_only( - c, - "excelreader_xlsx_parse_only", - xlsx_path(), - excelreader::XL_FORMAT_XLSX, - ); + bench_excelreader_parse_only(c, "excelreader_xlsx_parse_only", xlsx_path(), Format::Xlsx); } fn bench_excelreader_xlsb_parse_only(c: &mut Criterion) { - bench_excelreader_parse_only( - c, - "excelreader_xlsb_parse_only", - xlsb_path(), - excelreader::XL_FORMAT_XLSB, - ); + bench_excelreader_parse_only(c, "excelreader_xlsb_parse_only", xlsb_path(), Format::Xlsb); } criterion_group!( diff --git a/rust/excelreader/benches/parse_bench.rs b/rust/excelreader/benches/parse_bench.rs index 75f3ede1..0281ba52 100644 --- a/rust/excelreader/benches/parse_bench.rs +++ b/rust/excelreader/benches/parse_bench.rs @@ -1,5 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::ReadOptions; +use excelreader::{ExcelMapper, Workbook}; #[derive(Default, ExcelMapper)] struct Row { @@ -53,8 +54,11 @@ fn bench_parse_sheet(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let table = - parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("parse must succeed"); std::hint::black_box(table); }, criterion::BatchSize::SmallInput, @@ -68,7 +72,10 @@ fn bench_infer_schema(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 100) + let schema = workbook + .sheet(0) + .expect("sheet 0") + .infer_schema(1, 100, false) .expect("infer_schema must succeed"); std::hint::black_box(schema); }, @@ -93,8 +100,11 @@ fn bench_parse_sheet_large(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1) - .expect("parse_sheet must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("parse must succeed"); std::hint::black_box(table); }, criterion::BatchSize::SmallInput, @@ -108,7 +118,10 @@ fn bench_infer_schema_large(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 1000) + let schema = workbook + .sheet(0) + .expect("sheet 0") + .infer_schema(1, 1000, false) .expect("infer_schema must succeed"); std::hint::black_box(schema); }, @@ -117,15 +130,18 @@ fn bench_infer_schema_large(c: &mut Criterion) { }); } -fn bench_typed_chunks_large(c: &mut Criterion) { +fn bench_parse_batches_large(c: &mut Criterion) { let path = large_fixture_path(); for batch_size in [1_000_i64, 10_000, 0] { - c.bench_function(&format!("typed_chunks_large/{batch_size}"), |b| { + c.bench_function(&format!("parse_batches_large/{batch_size}"), |b| { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let chunks = workbook.sheet(0).expect("sheet 0").typed_chunks::(1, batch_size) - .expect("typed_chunks must succeed"); + let chunks = workbook + .sheet(0) + .expect("sheet 0") + .parse_batches::(&ReadOptions::new().batch_size(batch_size)) + .expect("parse_batches must succeed"); let mut rows = 0_i64; for batch in chunks { rows += batch.expect("a batch must read").len(); @@ -146,6 +162,6 @@ criterion_group!( bench_open_large, bench_parse_sheet_large, bench_infer_schema_large, - bench_typed_chunks_large + bench_parse_batches_large ); criterion_main!(benches); diff --git a/rust/excelreader/benches/write_bench.rs b/rust/excelreader/benches/write_bench.rs index f8c779bb..0740693f 100644 --- a/rust/excelreader/benches/write_bench.rs +++ b/rust/excelreader/benches/write_bench.rs @@ -18,9 +18,10 @@ //! Machine: state the CPU, OS and toolchain version alongside any number published from this file. use criterion::{criterion_group, criterion_main, Criterion}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; -use excelreader::writer::{write_columns, write_sheet, Column, OwnedColumn}; -use excelreader::{Date, XL_FORMAT_XLSX}; +use excelreader::ReadOptions; +use excelreader::{ + write_columns, write_sheet, Column, ColumnData, Date, ExcelMapper, Workbook, WriteOptions, +}; use std::path::{Path, PathBuf}; const FIXTURE: &str = "../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx"; @@ -53,7 +54,11 @@ fn load_rows() -> Vec { present in the repo" ); let workbook = Workbook::open(FIXTURE).expect("the fixture must open"); - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("the fixture must parse"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("the fixture must parse"); let rows: Vec = table.iter().collect(); assert!(!rows.is_empty(), "the fixture parsed to zero rows"); rows @@ -65,20 +70,49 @@ fn output_path(name: &str) -> PathBuf { path } +/// One string column's offsets and concatenated bytes. +fn strings<'a>(values: impl Iterator) -> (Vec, Vec) { + let mut offsets = vec![0]; + let mut data = Vec::new(); + for value in values { + data.extend_from_slice(value.as_bytes()); + offsets.push(data.len() as i32); + } + (offsets, data) +} + +fn str_column<'a>(name: &'a str, (offsets, data): &'a (Vec, Vec)) -> Column<'a> { + Column::new(Some(name), ColumnData::Str { offsets, data }) +} + fn benchmark_write(criterion: &mut Criterion) { let rows = load_rows(); - let owned: Vec = - ::to_columns(&rows).expect("transpose must succeed"); - let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + let region = strings(rows.iter().map(|r| r.region.as_str())); + let country = strings(rows.iter().map(|r| r.country.as_str())); + let item_type = strings(rows.iter().map(|r| r.item_type.as_str())); + let order_date: Vec = rows.iter().map(|r| r.order_date.days_since_epoch).collect(); + let order_id: Vec = rows.iter().map(|r| r.order_id).collect(); + let units_sold: Vec = rows.iter().map(|r| r.units_sold).collect(); + let total_revenue: Vec = rows.iter().map(|r| r.total_revenue).collect(); + let column = |name, data| Column::new(Some(name), data); + let borrowed = [ + str_column("Region", ®ion), + str_column("Country", &country), + str_column("Item Type", &item_type), + column("Order Date", ColumnData::Date(&order_date)), + column("Order ID", ColumnData::I64(&order_id)), + column("Units Sold", ColumnData::I64(&units_sold)), + column("Total Revenue", ColumnData::F64(&total_revenue)), + ]; + let options = WriteOptions::new(); let mut group = criterion.benchmark_group("write_xlsx_65k"); group.sample_size(20); group.bench_function("columns", |b| { let path = output_path("columns.xlsx"); - let target = path.to_str().expect("temp path must be UTF-8"); b.iter(|| { - write_columns(target, XL_FORMAT_XLSX, std::hint::black_box(&borrowed), None) + write_columns(&path, std::hint::black_box(&borrowed), &options) .expect("write_columns must succeed"); }); std::fs::remove_file(&path).ok(); @@ -86,9 +120,8 @@ fn benchmark_write(criterion: &mut Criterion) { group.bench_function("sheet", |b| { let path = output_path("sheet.xlsx"); - let target = path.to_str().expect("temp path must be UTF-8"); b.iter(|| { - write_sheet(target, XL_FORMAT_XLSX, std::hint::black_box(&rows), None) + write_sheet(&path, std::hint::black_box(&rows), &options) .expect("write_sheet must succeed"); }); std::fs::remove_file(&path).ok(); @@ -104,7 +137,9 @@ fn benchmark_write(criterion: &mut Criterion) { sheet.write_string(r, 0, &row.region).unwrap(); sheet.write_string(r, 1, &row.country).unwrap(); sheet.write_string(r, 2, &row.item_type).unwrap(); - sheet.write_number(r, 3, row.order_date.days_since_epoch as f64).unwrap(); + sheet + .write_number(r, 3, row.order_date.days_since_epoch as f64) + .unwrap(); sheet.write_number(r, 4, row.order_id as f64).unwrap(); sheet.write_number(r, 5, row.units_sold as f64).unwrap(); sheet.write_number(r, 6, row.total_revenue).unwrap(); @@ -118,4 +153,4 @@ fn benchmark_write(criterion: &mut Criterion) { } criterion_group!(benches, benchmark_write); -criterion_main!(benches); \ No newline at end of file +criterion_main!(benches); diff --git a/rust/excelreader/build.rs b/rust/excelreader/build.rs index 07415a99..4a628e0a 100644 --- a/rust/excelreader/build.rs +++ b/rust/excelreader/build.rs @@ -67,7 +67,12 @@ fn main() { && fs::read_to_string(&generated_def).is_ok_and(|old| old == new_def_content); if !up_to_date { generate_windows_implib( - &implib, &generated_def, &new_def_content, &target_env, &target_arch, &out_dir, + &implib, + &generated_def, + &new_def_content, + &target_env, + &target_arch, + &out_dir, &dll_basename, ); } diff --git a/rust/excelreader/src/aggregate.rs b/rust/excelreader/src/aggregate.rs index cfa85db8..5e76d743 100644 --- a/rust/excelreader/src/aggregate.rs +++ b/rust/excelreader/src/aggregate.rs @@ -7,13 +7,13 @@ //! can unwind into the library, and resumes that panic on the calling thread once the run has //! finished freeing every state. -use crate::{ - rows::RowRef, workbook, Error, XlCsvAggregation, XlCsvParallelOptions, XlRow, XL_OK, -}; +use crate::ffi::{self, XlCsvAggregation, XlRow, XL_OK}; +use crate::workbook::{self, len_i32, path_str}; +use crate::{Error, ErrorKind, OpenOptions, RowRef}; use std::any::Any; use std::ffi::c_void; use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Mutex; /// The status a shim returns to signal "a callback panicked". The library treats it like any other @@ -47,7 +47,8 @@ struct State { /// library later re-reads the partition, the failure is discarded with it and the run continues; /// otherwise the run ends with that code. A nonzero code from `combine`, or a panic from the seed /// closure, always ends the run. However it ends, the code is returned to the caller verbatim as -/// [`Error::code`]; use positive codes, since every code the library itself returns is `<= 0`. +/// [`ErrorKind::Callback`]; use positive codes, since every code the library itself returns is +/// `<= 0`. /// Sibling workers are never stopped, so both `accumulate` and `combine` must tolerate being called /// again after returning an error. pub trait CsvAccumulator: Send { @@ -58,30 +59,42 @@ pub trait CsvAccumulator: Send { fn combine(&mut self, other: &mut Self) -> Result<(), i32>; } -/// Mirrors `xl_csv_parallel_options`. Every zero field means "library default"; in particular -/// `degree_of_parallelism` 0 is the processor count and 1 forces a sequential run, and `header_row` -/// is 1-based with 0 meaning "no header". -#[derive(Clone, Copy, Debug, Default)] -pub struct CsvParallelOptions { - pub degree_of_parallelism: i32, - pub header_row: i32, - pub delimiter: i32, - pub quote: i32, - pub detect_bom: i32, - pub max_cell_bytes: i32, +/// What [`aggregate_csv`] reads: a CSV file, or CSV bytes already in memory. A `&str` converts to a +/// path, never to CSV text - pass `text.as_bytes()` for that. +#[derive(Clone, Copy, Debug)] +#[non_exhaustive] +pub enum CsvInput<'a> { + Path(&'a Path), + Bytes(&'a [u8]), } -impl CsvParallelOptions { - fn to_raw(self) -> XlCsvParallelOptions { - XlCsvParallelOptions { - struct_size: std::mem::size_of::() as i32, - degree_of_parallelism: self.degree_of_parallelism, - header_row: self.header_row, - delimiter: self.delimiter, - quote: self.quote, - detect_bom: self.detect_bom, - max_cell_bytes: self.max_cell_bytes, - } +impl<'a> From<&'a Path> for CsvInput<'a> { + fn from(path: &'a Path) -> Self { + CsvInput::Path(path) + } +} + +impl<'a> From<&'a PathBuf> for CsvInput<'a> { + fn from(path: &'a PathBuf) -> Self { + CsvInput::Path(path) + } +} + +impl<'a> From<&'a str> for CsvInput<'a> { + fn from(path: &'a str) -> Self { + CsvInput::Path(Path::new(path)) + } +} + +impl<'a> From<&'a [u8]> for CsvInput<'a> { + fn from(bytes: &'a [u8]) -> Self { + CsvInput::Bytes(bytes) + } +} + +impl<'a, const N: usize> From<&'a [u8; N]> for CsvInput<'a> { + fn from(bytes: &'a [u8; N]) -> Self { + CsvInput::Bytes(bytes) } } @@ -98,7 +111,10 @@ impl Shared<'_, A, F> { /// Keeps the FIRST payload: later panics from sibling workers are dropped rather than racing to /// overwrite the one that started the abort. fn store_panic(&self, payload: Box) -> i32 { - let mut slot = self.panic.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let mut slot = self + .panic + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); if slot.is_none() { *slot = Some(payload); } @@ -118,7 +134,10 @@ where match catch_unwind(AssertUnwindSafe(|| (shared.seed)())) { Ok(accumulator) => { unsafe { - *out_state = Box::into_raw(Box::new(State { value: accumulator, _nonzero: 0 })) as *mut c_void + *out_state = Box::into_raw(Box::new(State { + value: accumulator, + _nonzero: 0, + })) as *mut c_void }; XL_OK } @@ -179,9 +198,9 @@ where F: Fn() -> A, { let shared = unsafe { &*(user_data as *const Shared<'_, A, F>) }; - if let Err(payload) = - catch_unwind(AssertUnwindSafe(|| drop(unsafe { Box::from_raw(state as *mut State) }))) - { + if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { + drop(unsafe { Box::from_raw(state as *mut State) }) + })) { shared.store_panic(payload); } } @@ -202,21 +221,19 @@ where } fn shared(seed: &F) -> Shared<'_, A, F> { - Shared { seed, panic: Mutex::new(None), _marker: std::marker::PhantomData } -} - -/// The ABI takes every length as `int32`, so an oversized input is rejected rather than truncated -/// into a silently partial - but successful - aggregation. -fn length(len: usize, what: &str) -> Result { - i32::try_from(len).map_err(|_| { - Error::from_status( - crate::XL_INVALID_ARGUMENT, - format!("{what} is {len} bytes, past the ABI's int32 length limit"), - ) - }) + Shared { + seed, + panic: Mutex::new(None), + _marker: std::marker::PhantomData, + } } -/// Folds the CSV file at `path` into one `A` across several threads. +/// Folds a CSV file or buffer into one `A` across several threads. +/// +/// The CSV dialect (`csv_delimiter`, `csv_quote`, `csv_detect_bom`) and `max_cell_bytes` come from +/// `options`; its other settings do not apply. `header_row` is 1-based, `0` meaning no header. +/// `degree_of_parallelism` is `0` for every processor, `1` sequential, `n` up to `n` threads. +/// A buffer is not copied; the borrow keeps it valid for the call. /// /// `seed` is called at least once per partition, on worker threads, and may be called again for a /// partition that has to be re-read - a seeded accumulator can therefore be dropped without ever @@ -226,85 +243,66 @@ fn length(len: usize, what: &str) -> Result { /// The call blocks until the run finishes, so `seed` may borrow locals freely: there is no `'static` /// bound and none is needed. A panic inside any callback aborts the run and is resumed here, on the /// calling thread, once the library has freed every state. -pub fn aggregate_csv_file( - path: &Path, - seed: F, - options: &CsvParallelOptions, -) -> Result -where - A: CsvAccumulator, - F: Fn() -> A + Sync, -{ - workbook::check_abi_version()?; - let bytes = path - .to_str() - .ok_or_else(|| { - Error::from_status( - crate::XL_INVALID_ARGUMENT, - format!("path {} is not valid UTF-8", path.display()), - ) - })? - .as_bytes(); - let path_len = length(bytes.len(), "path")?; - let shared = shared::(&seed); - let agg = raw_aggregation(&shared); - let raw_options = options.to_raw(); - let mut state: *mut c_void = std::ptr::null_mut(); - - let status = unsafe { - crate::xl_csv_aggregate_file( - bytes.as_ptr(), - path_len, - &agg, - &raw_options, - &mut state, - ) - }; - finish(shared, status, state) -} - -/// Folds an in-memory CSV buffer into one `A`. The buffer is not copied and must stay valid for the -/// duration of the call, which this signature guarantees. Otherwise identical to -/// [`aggregate_csv_file`]. -pub fn aggregate_csv_memory( - data: &[u8], +pub fn aggregate_csv<'a, A, F>( + input: impl Into>, + options: &OpenOptions, + header_row: i32, + degree_of_parallelism: i32, seed: F, - options: &CsvParallelOptions, ) -> Result where A: CsvAccumulator, F: Fn() -> A + Sync, { workbook::check_abi_version()?; - aggregate_csv_memory_unchecked(data, seed, options) + aggregate_unchecked( + input.into(), + options, + header_row, + degree_of_parallelism, + seed, + ) } -/// [`aggregate_csv_memory`] without the ABI check, for the warm-up run in +/// [`aggregate_csv`] without the ABI check, for the warm-up run in /// [`workbook::check_abi_version`] itself - calling the public entry point from there would /// re-enter its `OnceLock` and deadlock. -pub(crate) fn aggregate_csv_memory_unchecked( - data: &[u8], +pub(crate) fn aggregate_unchecked( + input: CsvInput<'_>, + options: &OpenOptions, + header_row: i32, + degree_of_parallelism: i32, seed: F, - options: &CsvParallelOptions, ) -> Result where A: CsvAccumulator, F: Fn() -> A + Sync, { - let data_len = length(data.len(), "buffer")?; + let raw_options = options.to_csv_parallel_raw(header_row, degree_of_parallelism); let shared = shared::(&seed); let agg = raw_aggregation(&shared); - let raw_options = options.to_raw(); let mut state: *mut c_void = std::ptr::null_mut(); - let status = unsafe { - crate::xl_csv_aggregate_memory( - data.as_ptr(), - data_len, - &agg, - &raw_options, - &mut state, - ) + let status = match input { + CsvInput::Path(path) => { + let path = path_str(path)?; + let path_len = len_i32(path.len(), "path")?; + unsafe { + ffi::xl_csv_aggregate_file(path.as_ptr(), path_len, &agg, &raw_options, &mut state) + } + } + CsvInput::Bytes(data) => { + let data_len = len_i32(data.len(), "buffer")?; + unsafe { + ffi::xl_csv_aggregate_memory( + data.as_ptr(), + data_len, + &agg, + &raw_options, + &mut state, + ) + } + } }; finish(shared, status, state) } @@ -333,8 +331,10 @@ where }; if status == PANIC_STATUS { - if let Some(payload) = - shared.panic.into_inner().unwrap_or_else(|poisoned| poisoned.into_inner()) + if let Some(payload) = shared + .panic + .into_inner() + .unwrap_or_else(|poisoned| poisoned.into_inner()) { drop(winner); resume_unwind(payload); @@ -344,9 +344,9 @@ where return Err(status_error(status)); } winner.map(|boxed| boxed.value).ok_or_else(|| { - Error::from_status( - crate::XL_ERROR, - "native reported success but wrote no aggregation state".to_string(), + Error::new( + ErrorKind::Other, + "native reported success but wrote no aggregation state", ) }) } @@ -356,7 +356,10 @@ where /// message for. fn status_error(status: i32) -> Error { if status > 0 { - Error::from_status(status, format!("an aggregation callback returned status {status}")) + Error::new( + ErrorKind::Callback(status), + format!("an aggregation callback returned status {status}"), + ) } else { workbook::last_error(status) } diff --git a/rust/excelreader/src/arrow.rs b/rust/excelreader/src/arrow.rs index 72130276..cbf54060 100644 --- a/rust/excelreader/src/arrow.rs +++ b/rust/excelreader/src/arrow.rs @@ -13,63 +13,89 @@ use arrow::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use arrow::record_batch::RecordBatchReader; -use crate::workbook::{build_specs, check, ExcelMapper, Sheet, Workbook}; -use crate::{Error, XL_ERROR}; - -/// Schema-driven parse of one sheet into an Arrow [`RecordBatch`], using the same -/// `#[derive(ExcelMapper)]` mapping as [`crate::workbook::parse_sheet`]. -/// -/// `header_row` has the same meaning as in `parse_sheet` (0 = no header). -pub fn parse_arrow(sheet: Sheet<'_>, header_row: i32) -> Result { - parse_arrow_parallel::(sheet, header_row, 1) -} +use crate::ffi; +use crate::workbook::{build_specs, check}; +use crate::{Error, ErrorKind, ExcelMapper, ReadOptions, Sheet, Workbook}; + +impl<'w> Sheet<'w> { + /// [`parse`](Self::parse) into one Arrow [`RecordBatch`], with the same `#[derive(ExcelMapper)]` + /// mapping and [`ReadOptions`]. + pub fn to_arrow(&self, options: &ReadOptions) -> Result { + let arena = build_specs::(); + + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + + check(unsafe { + ffi::xl_parse_arrow( + self.handle(), + self.index(), + arena.specs.as_ptr(), + arena.specs.len() as i32, + options.header_row, + options.parallelism, + &mut array as *mut FFI_ArrowArray as *mut c_void, + &mut schema as *mut FFI_ArrowSchema as *mut c_void, + ) + })?; + + let data = unsafe { from_ffi(array, &schema) }.map_err(|e| { + Error::new( + ErrorKind::Other, + format!("importing the native Arrow array failed: {e}"), + ) + })?; + + Ok(RecordBatch::from(StructArray::from(data))) + } -/// [`parse_arrow`] with a degree of parallelism: `0` uses every processor, `1` is sequential, `n` -/// up to `n` threads, negative is an error. Only CSV is split; the batch equals `parse_arrow`'s. -pub fn parse_arrow_parallel( - sheet: Sheet<'_>, - header_row: i32, - degree_of_parallelism: i32, -) -> Result { - let arena = build_specs::(); - - let mut array = FFI_ArrowArray::empty(); - let mut schema = FFI_ArrowSchema::empty(); - - check(unsafe { - crate::xl_parse_arrow( - sheet.handle(), - sheet.index(), - arena.specs.as_ptr(), - arena.specs.len() as i32, - header_row, - degree_of_parallelism, - &mut array as *mut FFI_ArrowArray as *mut c_void, - &mut schema as *mut FFI_ArrowSchema as *mut c_void, - ) - })?; - - let data = unsafe { from_ffi(array, &schema) }.map_err(|e| { - Error::from_status( - XL_ERROR, - format!("importing the native Arrow array failed: {e}"), - ) - })?; - - Ok(RecordBatch::from(StructArray::from(data))) + /// [`to_arrow`](Self::to_arrow) delivered [`ReadOptions`]'s batch size rows at a time; a + /// negative batch size is an error. Any number of streams may be open on one workbook. + pub fn to_arrow_stream( + &self, + options: &ReadOptions, + ) -> Result, Error> { + let arena = build_specs::(); + + let mut stream = FFI_ArrowArrayStream::empty(); + + check(unsafe { + ffi::xl_parse_arrow_stream( + self.handle(), + self.index(), + arena.specs.as_ptr(), + arena.specs.len() as i32, + options.header_row, + options.batch_size, + &mut stream as *mut FFI_ArrowArrayStream as *mut c_void, + ) + })?; + + let inner = ArrowArrayStreamReader::try_new(stream).map_err(|e| { + Error::new( + ErrorKind::Other, + format!("importing the native Arrow stream failed: {e}"), + ) + })?; + + Ok(ArrowBatches { + inner, + _workbook: PhantomData, + }) + } } -/// Batched counterpart to [`parse_arrow`], one [`RecordBatch`] at a time. +/// Batches of [`Sheet::to_arrow_stream`], one [`RecordBatch`] at a time. /// -/// Wraps arrow-rs's `'static` reader only to carry the `'a` borrow, which keeps the -/// workbook alive for at least as long as the stream. Unlike [`crate::workbook::TypedChunks`] it does -/// not fuse after an error, so break on the first `Err` rather than spinning on the latched one. -pub struct ArrowChunks<'a> { +/// Wraps arrow-rs's `'static` reader only to carry the `'a` borrow, which keeps the workbook alive +/// for at least as long as the stream. Unlike [`TypedBatches`](crate::TypedBatches) it does not fuse +/// after an error, so break on the first `Err` rather than spinning on the latched one. +pub struct ArrowBatches<'a> { inner: ArrowArrayStreamReader, _workbook: PhantomData<&'a Workbook>, } -impl Iterator for ArrowChunks<'_> { +impl Iterator for ArrowBatches<'_> { type Item = Result; fn next(&mut self) -> Option { @@ -77,44 +103,8 @@ impl Iterator for ArrowChunks<'_> { } } -impl RecordBatchReader for ArrowChunks<'_> { +impl RecordBatchReader for ArrowBatches<'_> { fn schema(&self) -> SchemaRef { self.inner.schema() } } - -/// `parse_arrow` delivered a batch at a time. `batch_size` is rows per batch, 0 unbounded, -/// negative an error. Any number of streams may be open on one workbook. -pub fn parse_arrow_stream( - sheet: Sheet<'_>, - header_row: i32, - batch_size: i64, -) -> Result, Error> { - let arena = build_specs::(); - - let mut stream = FFI_ArrowArrayStream::empty(); - - check(unsafe { - crate::xl_parse_arrow_stream( - sheet.handle(), - sheet.index(), - arena.specs.as_ptr(), - arena.specs.len() as i32, - header_row, - batch_size, - &mut stream as *mut FFI_ArrowArrayStream as *mut c_void, - ) - })?; - - let inner = ArrowArrayStreamReader::try_new(stream).map_err(|e| { - Error::from_status( - XL_ERROR, - format!("importing the native Arrow stream failed: {e}"), - ) - })?; - - Ok(ArrowChunks { - inner, - _workbook: PhantomData, - }) -} diff --git a/rust/excelreader/src/error.rs b/rust/excelreader/src/error.rs index e3d0ac2e..88591d1e 100644 --- a/rust/excelreader/src/error.rs +++ b/rust/excelreader/src/error.rs @@ -1,67 +1,96 @@ use std::fmt; -/// A native ExcelReader error. `code` is one of the `XL_*` status constants; `message` is the -/// detail from `xl_last_error_ptr` on the calling thread at the time of failure. -/// -/// [`PasswordRequired`](Error::PasswordRequired) and [`PasswordIncorrect`](Error::PasswordIncorrect) -/// get their own variants - rather than staying folded into a generic `code`/`message` pair - so -/// callers can `match`/`matches!` on "needs a password" without hardcoding the `-6`/`-7` status -/// constants themselves. Every other status stays in [`Native`](Error::Native), which is where the -/// distinction is drawn: not "every code gets a variant" but "every code a caller needs to branch on -/// by name gets one". -#[derive(Debug, Clone)] -pub enum Error { - /// Any native status not covered by a more specific variant below. - Native { code: i32, message: String }, - /// The workbook is encrypted and no password was supplied (`XL_STATUS_PASSWORD_REQUIRED`). - PasswordRequired { message: String }, - /// The supplied password did not match the workbook's verifier (`XL_STATUS_PASSWORD_INCORRECT`). - PasswordIncorrect { message: String }, +use crate::ffi; + +/// What went wrong, for callers that branch on the failure. The human-readable detail is +/// [`Error::message`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ErrorKind { + /// An argument was rejected before any work was done: a bad option value, an out-of-range + /// index, mismatched column lengths, a path that is not UTF-8. + InvalidArgument, + /// The workbook is encrypted and no password was supplied. + PasswordRequired, + /// The supplied password did not match the workbook's verifier. + PasswordIncorrect, + /// A [`CsvAccumulator`](crate::CsvAccumulator) callback returned this code, passed through + /// verbatim. + Callback(i32), + /// Anything else: an I/O failure, a malformed or unsupported file, a resource limit. + Other, +} + +/// An ExcelReader error: an [`ErrorKind`] plus the detail message the native library (or this +/// crate) reported at the time of failure. +#[derive(Clone, Debug)] +pub struct Error { + kind: ErrorKind, + message: String, } impl Error { - /// Builds an [`Error`] from a raw `XL_*` status code and a detail message, routing - /// `XL_STATUS_PASSWORD_REQUIRED`/`XL_STATUS_PASSWORD_INCORRECT` to their own variants and - /// everything else to [`Native`](Error::Native). The one place that mapping happens, so every - /// call site that turns a status code into an `Error` (here, in `arrow`/`writer`, and in - /// `excelreader_derive`'s generated `append_tokens`) goes through it instead of re-deriving the - /// mapping. `pub`, not `pub(crate)`: the derive macro expands into the crate that invokes it, - /// not this one, so its generated code needs a public constructor to build a well-formed `Error` - /// without reaching into a private field or duplicating this match. - #[must_use] - pub fn from_status(code: i32, message: String) -> Self { - match code { - crate::XL_STATUS_PASSWORD_REQUIRED => Error::PasswordRequired { message }, - crate::XL_STATUS_PASSWORD_INCORRECT => Error::PasswordIncorrect { message }, - _ => Error::Native { code, message }, + pub(crate) fn new(kind: ErrorKind, message: impl Into) -> Self { + Error { + kind, + message: message.into(), } } - /// The `XL_*` status constant this error carries. + /// The one place a native status code becomes an [`ErrorKind`]. Positive codes only ever come + /// from an aggregation callback; every library status is `<= 0`. + pub(crate) fn from_status(code: i32, message: String) -> Self { + let kind = match code { + ffi::XL_INVALID_ARGUMENT => ErrorKind::InvalidArgument, + ffi::XL_STATUS_PASSWORD_REQUIRED => ErrorKind::PasswordRequired, + ffi::XL_STATUS_PASSWORD_INCORRECT => ErrorKind::PasswordIncorrect, + code if code > 0 => ErrorKind::Callback(code), + _ => ErrorKind::Other, + }; + Error { kind, message } + } + + pub(crate) fn io(error: &std::io::Error) -> Self { + Error::new(ErrorKind::Other, error.to_string()) + } + #[must_use] - pub fn code(&self) -> i32 { - match self { - Error::Native { code, .. } => *code, - Error::PasswordRequired { .. } => crate::XL_STATUS_PASSWORD_REQUIRED, - Error::PasswordIncorrect { .. } => crate::XL_STATUS_PASSWORD_INCORRECT, - } + pub fn kind(&self) -> ErrorKind { + self.kind } - /// The detail from `xl_last_error_ptr` at the time of failure. + /// The detail reported at the time of failure. #[must_use] pub fn message(&self) -> &str { - match self { - Error::Native { message, .. } - | Error::PasswordRequired { message } - | Error::PasswordIncorrect { message } => message, - } + &self.message } } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ExcelReader error {}: {}", self.code(), self.message()) + write!(f, "ExcelReader error ({:?}): {}", self.kind, self.message) } } impl std::error::Error for Error {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn statuses_map_onto_kinds() { + let kind = |code| Error::from_status(code, String::new()).kind(); + assert_eq!(kind(ffi::XL_INVALID_ARGUMENT), ErrorKind::InvalidArgument); + assert_eq!( + kind(ffi::XL_STATUS_PASSWORD_REQUIRED), + ErrorKind::PasswordRequired + ); + assert_eq!( + kind(ffi::XL_STATUS_PASSWORD_INCORRECT), + ErrorKind::PasswordIncorrect + ); + assert_eq!(kind(42), ErrorKind::Callback(42)); + assert_eq!(kind(-5), ErrorKind::Other); + } +} diff --git a/rust/excelreader/src/ffi.rs b/rust/excelreader/src/ffi.rs new file mode 100644 index 00000000..8cdc9fa7 --- /dev/null +++ b/rust/excelreader/src/ffi.rs @@ -0,0 +1,495 @@ +//! Raw FFI bindings to ExcelReader's C ABI (`excelreader.h`). Crate-private: every public type is a +//! safe wrapper over these. + +use std::os::raw::{c_int, c_void}; + +pub const XL_OK: i32 = 0; +pub const XL_EOF: i32 = -1; +pub const XL_BUFFER_TOO_SMALL: i32 = -2; +pub const XL_INVALID_ARGUMENT: i32 = -4; +/// The workbook is encrypted and no password was supplied. +pub const XL_STATUS_PASSWORD_REQUIRED: i32 = -6; +/// The supplied password did not match the workbook's verifier. +pub const XL_STATUS_PASSWORD_INCORRECT: i32 = -7; + +pub const XL_CELL_EMPTY: i32 = 0; +pub const XL_CELL_STRING: i32 = 1; +pub const XL_CELL_NUMBER: i32 = 2; +pub const XL_CELL_DATE: i32 = 3; +pub const XL_CELL_BOOL: i32 = 4; +pub const XL_CELL_FORMULA: i32 = 5; +pub const XL_CELL_ERROR: i32 = 6; + +/// ABI revision this crate is compiled against. Every constructor refuses to proceed when the loaded +/// library's `xl_abi_version()` disagrees - see `workbook::check_abi_version`. +pub const XL_ABI_VERSION: i32 = 6; + +pub const XL_T_STRING: i32 = 0; +pub const XL_T_I64: i32 = 1; +pub const XL_T_F64: i32 = 2; +pub const XL_T_BOOL: i32 = 3; +/// Days since 1970-01-01, stored as `i32`. +pub const XL_T_DATE: i32 = 4; +/// Microseconds since midnight, stored as `i64`. +pub const XL_T_TIME: i32 = 5; +/// Microseconds since 1970-01-01T00:00:00Z, stored as `i64`. +pub const XL_T_TIMESTAMP: i32 = 6; + +/// `xl_infer_schema` flag: also type cells that hold text (integers, decimals, booleans, ISO dates). +pub const XL_INFER_PARSE_TEXT: i32 = 1; + +pub const XL_SHEET_VISIBLE: i32 = 0; +pub const XL_SHEET_HIDDEN: i32 = 1; +pub const XL_SHEET_VERY_HIDDEN: i32 = 2; + +pub const XL_FORMAT_AUTO: i32 = 0; +pub const XL_FORMAT_XLS: i32 = 1; +pub const XL_FORMAT_XLSX: i32 = 2; +pub const XL_FORMAT_XLSB: i32 = 3; +pub const XL_FORMAT_CSV: i32 = 4; + +/// Every boolean-shaped `xl_open_options` field uses one of these three states, never a plain 0/1 - +/// several of them default to true, so a bare 0 would be ambiguous between "off" and "use the +/// library default". +pub const XL_OPT_DEFAULT: i32 = 0; +pub const XL_OPT_FALSE: i32 = 1; +pub const XL_OPT_TRUE: i32 = 2; + +/// Opaque handle - never dereferenced by Rust, only passed back to `xl_*` functions. +#[repr(C)] +pub struct XlWorkbook { + _private: [u8; 0], +} + +/// Opaque row cursor (`xl_rows`) - never dereferenced by Rust, only passed back to `xl_rows_*`. +#[repr(C)] +pub struct XlRows { + _private: [u8; 0], +} + +#[repr(C)] +pub struct XlTypedReader { + _private: [u8; 0], +} + +/// Mirrors `xl_open_options`. `struct_size` must be set to `size_of::()`; build one +/// through `OpenOptions` rather than by hand. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlOpenOptions { + pub struct_size: i32, + pub csv_sniff_dialect: i32, + pub csv_delimiter: i32, + pub csv_quote: i32, + pub csv_detect_bom: i32, + pub csv_max_cell_bytes: i32, + pub csv_intern_strings: i32, + pub max_total_decompressed_bytes: i64, + pub max_cell_bytes: i32, + pub max_shared_string_bytes: i64, + pub max_zip_entries: i32, + pub prefetch_decompression: i32, + pub intern_strings: i32, + + /// Password for an encrypted OOXML workbook, as UTF-8 bytes. NULL (with `password_len` 0) means + /// "not encrypted, or fail with `XL_STATUS_PASSWORD_REQUIRED`". Not NUL-terminated - + /// `password_len` is authoritative, since a password may contain any byte. The pointer need only + /// remain valid for the duration of the call - see `options::OpenOptionsRaw`. + pub password: *const u8, + pub password_len: i32, + pub source_cache_bytes: i64, + pub max_buffered_bytes: i64, + pub source_block_size: i32, + pub verify_zip_checksums: i32, +} + +#[repr(C)] +pub struct XlSource { + pub struct_size: i32, + pub user_data: *mut c_void, + pub length: i64, + pub read_at: Option i64>, + pub release: Option, +} + +#[repr(C)] +pub struct XlStream { + pub struct_size: i32, + pub user_data: *mut c_void, + pub read: Option i64>, + pub release: Option, +} + +/// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with +/// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts +/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. +/// Build one through `WriteOptions` rather than by hand. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlWriteOptions { + pub struct_size: i32, + pub sheet_name_len: i32, + pub sheet_name: *const u8, + pub csv_delimiter: i32, + pub csv_quote: i32, + pub date1904: i32, + pub use_shared_strings: i32, +} + +#[repr(C)] +pub struct XlColumnSpec { + pub names: *const *const u8, + pub name_lens: *const i32, + pub name_count: i32, + pub index: i32, + pub r#type: i32, + pub nullable: i32, +} + +/// One column of a native table. Reachable from `__private` because the derive's generated +/// bindings read it through the `column_*` accessors. +#[repr(C)] +pub struct XlColumn { + pub r#type: i32, + pub length: i64, + pub values: *const c_void, + pub validity: *const u8, + pub data: *const u8, + pub data_len: i64, +} + +#[repr(C)] +pub struct XlTable { + pub column_count: i32, + pub row_count: i64, + pub columns: *mut XlColumn, +} + +/// Mirrors `xl_inferred_schema`. Owned by the native library until `xl_free_schema`. +#[repr(C)] +pub struct XlInferredSchema { + pub columns: *mut XlColumnSpec, + pub column_count: i32, +} + +/// Mirrors `xl_buffer`: an owned block of native memory returned by `xl_write_typed_to_memory` or +/// `xl_write_handle_bytes`. Released with `xl_free_buffer` - see `workbook::buffer_to_vec`, the one +/// place this crate touches the raw struct directly. +#[repr(C)] +pub struct XlBuffer { + pub data: *mut u8, + pub len: i64, +} + +/// Opaque streaming writer handle - never dereferenced by Rust, only passed back to `xl_*` +/// functions. +#[repr(C)] +pub struct XlWriterHandle { + _private: [u8; 0], +} + +/// Mirrors `xl_row_cell`. Named `cell_type` because `type` is a Rust keyword. +#[repr(C)] +#[derive(Debug)] +pub struct XlRowCell { + pub column: i32, + pub cell_type: i32, + pub value_len: i32, + pub value: *const u8, +} + +/// Mirrors `xl_row`. +#[repr(C)] +pub struct XlRow { + pub cell_count: i32, + pub cells: *mut XlRowCell, +} + +/// Mirrors `xl_csv_aggregation`. Every callback runs on a library worker thread except +/// `free_state`, which runs on the calling thread after the run has finished; none may let a +/// panic escape into the library - build one through `aggregate` rather than by hand. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XlCsvAggregation { + pub struct_size: i32, + pub seed: Option i32>, + pub accumulate: Option i32>, + pub combine: Option i32>, + pub free_state: Option, + pub user_data: *mut c_void, +} + +/// Mirrors `xl_csv_parallel_options`. `struct_size` must be set to +/// `size_of::()`; built by `OpenOptions::to_csv_parallel_raw`. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlCsvParallelOptions { + pub struct_size: i32, + pub degree_of_parallelism: i32, + pub header_row: i32, + pub delimiter: i32, + pub quote: i32, + pub detect_bom: i32, + pub max_cell_bytes: i32, +} + +extern "C" { + pub fn xl_abi_version() -> c_int; + + pub fn xl_open_file( + path: *const u8, + path_len: i32, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_memory( + data: *const u8, + data_len: i32, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_source( + source: *const XlSource, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_stream( + stream: *const XlStream, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_set_source_error(message: *const u8, len: i32); + + pub fn xl_close(handle: *mut XlWorkbook) -> c_int; + + pub fn xl_sheet_count(handle: *mut XlWorkbook, out_count: *mut i32) -> c_int; + + pub fn xl_sheet_name_at( + handle: *mut XlWorkbook, + index: i32, + buffer: *mut u8, + capacity: i32, + out_len: *mut i32, + ) -> c_int; + + pub fn xl_sheet_visibility_at( + handle: *mut XlWorkbook, + index: i32, + out_visibility: *mut i32, + ) -> c_int; + + /// A name that matches no sheet is `XL_OK` with `-1`. + pub fn xl_sheet_index( + handle: *mut XlWorkbook, + name: *const u8, + name_len: i32, + out_index: *mut i32, + ) -> c_int; + + pub fn xl_is_date1904(handle: *mut XlWorkbook, out_flag: *mut i32) -> c_int; + + pub fn xl_parse_typed( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + degree_of_parallelism: i32, + out_table: *mut XlTable, + ) -> c_int; + + /// Same schema-driven parse as `xl_parse_typed`, exported as one top-level Arrow struct + /// array/schema. `out_array`/`out_schema` are `struct ArrowArray*`/`struct ArrowSchema*` from + /// the Arrow C Data Interface, typed here as `c_void` because arrow-rs's own `#[repr(C)]` + /// `FFI_ArrowArray`/`FFI_ArrowSchema` are ABI-identical to them - redeclaring the spec structs + /// would be a second source of truth for a fixed, versioned ABI. + /// + /// On `XL_OK` the caller owns both and releases each through its OWN `release` callback, never + /// through `xl_free_table`. On any other status both outputs are left untouched. + #[cfg(feature = "arrow")] + pub fn xl_parse_arrow( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + degree_of_parallelism: i32, + out_array: *mut c_void, + out_schema: *mut c_void, + ) -> c_int; + + /// `out_stream` is a `struct ArrowArrayStream*`, typed as `c_void` for the same reason + /// `xl_parse_arrow`'s outputs are. Its `release` is what closes the underlying read. + #[cfg(feature = "arrow")] + pub fn xl_parse_arrow_stream( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + max_rows: i64, + out_stream: *mut c_void, + ) -> c_int; + + pub fn xl_free_table(table: *mut XlTable); + + /// `max_rows` is rows per batch, 0 meaning one unbounded batch. + pub fn xl_typed_reader_open( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + max_rows: i64, + out_reader: *mut *mut XlTypedReader, + ) -> c_int; + + /// `XL_OK` with a batch, or `XL_EOF` at end of sheet. A failure latches: every later call + /// repeats it. + pub fn xl_typed_reader_next(reader: *mut XlTypedReader, out_table: *mut XlTable) -> c_int; + + pub fn xl_typed_reader_close(reader: *mut XlTypedReader); + + pub fn xl_infer_schema( + handle: *mut XlWorkbook, + sheet: i32, + header_row: i32, + sample_size: i32, + flags: i32, + out_schema: *mut XlInferredSchema, + ) -> c_int; + + pub fn xl_free_schema(schema: *mut XlInferredSchema); + pub fn xl_write_typed( + path: *const u8, + path_len: i32, + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + ) -> c_int; + + /// Same as `xl_write_typed`, except the result is returned as `out_buffer` instead of being + /// written to a path. Only read `*out_buffer` when the call returns `XL_OK`; on failure it is + /// zeroed, and `xl_free_buffer` on a zeroed buffer is a no-op. + pub fn xl_write_typed_to_memory( + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + out_buffer: *mut XlBuffer, + ) -> c_int; + + /// Releases a buffer returned by `xl_write_typed_to_memory` or `xl_write_handle_bytes` and + /// resets it to zero. Safe on a zeroed value. + pub fn xl_free_buffer(buffer: *mut XlBuffer); + + /// Reads the plaintext XLSX/XLSB package at `package_path` and writes its agile-encrypted + /// (ECMA-376 4.4) counterpart to `destination_path`, overwriting an existing file. The result + /// opens with the same password via `xl_open_file`'s `xl_open_options::password`. + pub fn xl_encrypt_package( + package_path: *const u8, + package_path_len: i32, + destination_path: *const u8, + destination_path_len: i32, + password: *const u8, + password_len: i32, + ) -> c_int; + + pub fn xl_open_write_handle( + path: *const u8, + path_len: i32, + format: i32, + options: *const XlWriteOptions, + out_handle: *mut *mut XlWriterHandle, + ) -> c_int; + + pub fn xl_open_write_handle_to_memory( + format: i32, + options: *const XlWriteOptions, + out_handle: *mut *mut XlWriterHandle, + ) -> c_int; + + pub fn xl_start_sheet(handle: *mut XlWriterHandle, name: *const u8, name_len: i32) -> c_int; + pub fn xl_start_row(handle: *mut XlWriterHandle) -> c_int; + + pub fn xl_write_string(handle: *mut XlWriterHandle, value: *const u8, value_len: i32) -> c_int; + pub fn xl_write_int64(handle: *mut XlWriterHandle, value: i64) -> c_int; + pub fn xl_write_float64(handle: *mut XlWriterHandle, value: f64) -> c_int; + pub fn xl_write_bool(handle: *mut XlWriterHandle, value: i32) -> c_int; + pub fn xl_write_date(handle: *mut XlWriterHandle, days_since_epoch: i32) -> c_int; + pub fn xl_write_time(handle: *mut XlWriterHandle, micros_since_midnight: i64) -> c_int; + pub fn xl_write_timestamp(handle: *mut XlWriterHandle, micros_since_epoch: i64) -> c_int; + pub fn xl_write_null(handle: *mut XlWriterHandle, r#type: i32) -> c_int; + + pub fn xl_end_row(handle: *mut XlWriterHandle) -> c_int; + pub fn xl_end_sheet(handle: *mut XlWriterHandle) -> c_int; + pub fn xl_close_write_handle(handle: *mut XlWriterHandle) -> c_int; + + /// Reads back everything written so far to a handle opened by `xl_open_write_handle_to_memory`. + /// `XL_INVALID_ARGUMENT` for one from `xl_open_write_handle`. Only read `*out_buffer` when the + /// call returns `XL_OK`; on failure it is zeroed, and `xl_free_buffer` on a zeroed buffer is a + /// no-op. + pub fn xl_write_handle_bytes(handle: *mut XlWriterHandle, out_buffer: *mut XlBuffer) -> c_int; + + pub fn xl_rows_open(handle: *mut XlWorkbook, sheet: i32, out_rows: *mut *mut XlRows) -> c_int; + pub fn xl_rows_close(rows: *mut XlRows) -> c_int; + pub fn xl_rows_next( + rows: *mut XlRows, + buffer: *mut u8, + capacity: i32, + out_written: *mut i32, + ) -> c_int; + pub fn xl_rows_read_all_blob( + rows: *mut XlRows, + buffer: *mut u8, + capacity: i32, + out_written: *mut i32, + ) -> c_int; + + /// Folds a CSV file into one caller-owned accumulator across several threads. Blocks until the + /// run finishes; the surviving state is written to `out_state` only on `XL_OK` and becomes the + /// caller's to free. See the contract above `xl_csv_aggregation` in `excelreader.h`, and + /// `aggregate::aggregate_csv` for the safe wrapper. + pub fn xl_csv_aggregate_file( + path: *const u8, + path_len: i32, + agg: *const XlCsvAggregation, + options: *const XlCsvParallelOptions, + out_state: *mut *mut c_void, + ) -> c_int; + + /// Same as `xl_csv_aggregate_file` over an in-memory buffer, which must stay valid and + /// unmodified until the call returns - the library does not copy it. + pub fn xl_csv_aggregate_memory( + data: *const u8, + data_len: i32, + agg: *const XlCsvAggregation, + options: *const XlCsvParallelOptions, + out_state: *mut *mut c_void, + ) -> c_int; + + pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn abi_version_matches_the_loaded_library() { + assert_eq!( + unsafe { xl_abi_version() }, + XL_ABI_VERSION, + "the linked native library speaks a different ABI revision than this crate" + ); + } +} diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 64e81bc5..91a60ce8 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -1,515 +1,55 @@ -// Raw FFI bindings to ExcelReader's C ABI (`excelreader.h`). The `extern "C"` block below is -// declared directly in this crate root (not a submodule), so `crate::xl_open_file` etc. resolve -// from anywhere in the crate. See the `workbook` module for the safe wrapper built on top of this. #![doc = include_str!("../README.md")] #![doc( html_logo_url = "https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png", html_favicon_url = "https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png" )] -pub mod aggregate; +mod aggregate; mod error; +pub(crate) mod ffi; mod options; +mod rows; +mod source; mod temporal; -pub mod workbook; -pub mod writer; -pub mod writer_handle; -pub mod rows; -pub mod source; +mod workbook; +mod writer; +mod writer_handle; #[cfg(feature = "arrow")] -pub mod arrow; +mod arrow; -pub use aggregate::{aggregate_csv_file, aggregate_csv_memory, CsvAccumulator, CsvParallelOptions}; -pub use error::Error; -pub use rows::{AllRows, CellIter, CellRef, CellType, DecodedRows, RowCursor, RowRef}; +pub use aggregate::{aggregate_csv, CsvAccumulator, CsvInput}; +pub use error::{Error, ErrorKind}; +pub use options::{Format, OpenOptions, ReadOptions, WriteOptions}; +pub use rows::{AllRows, CellIter, CellRef, CellType, RowCursor, RowRef}; pub use source::Source; -pub use options::{OpenOptions, OpenOptionsRaw, WriteOptions}; pub use temporal::{Date, Time, Timestamp}; +pub use workbook::{ + ColumnType, ExcelMapper, InferredColumn, Sheet, SheetVisibility, TableView, TableViewIter, + TypedBatches, Workbook, +}; +pub use writer::{ + encrypt_package, write_columns, write_columns_to, write_sheet, write_sheet_to, Column, + ColumnData, ExcelWriter, +}; +pub use writer_handle::WriterHandle; -use std::os::raw::{c_int, c_void}; - -pub const XL_OK: i32 = 0; -pub const XL_EOF: i32 = -1; -pub const XL_BUFFER_TOO_SMALL: i32 = -2; -pub const XL_INVALID_HANDLE: i32 = -3; -pub const XL_INVALID_ARGUMENT: i32 = -4; -pub const XL_ERROR: i32 = -5; -/// The workbook is encrypted and no password was supplied. -pub const XL_STATUS_PASSWORD_REQUIRED: i32 = -6; -/// The supplied password did not match the workbook's verifier. -pub const XL_STATUS_PASSWORD_INCORRECT: i32 = -7; - -pub const XL_CELL_EMPTY: i32 = 0; -pub const XL_CELL_STRING: i32 = 1; -pub const XL_CELL_NUMBER: i32 = 2; -pub const XL_CELL_DATE: i32 = 3; -pub const XL_CELL_BOOL: i32 = 4; -pub const XL_CELL_FORMULA: i32 = 5; -pub const XL_CELL_ERROR: i32 = 6; - -/// ABI revision this crate is compiled against. `Workbook::open` refuses to proceed when the loaded -/// library's `xl_abi_version()` disagrees - see `workbook::check_abi_version`. -pub const XL_ABI_VERSION: i32 = 6; - -pub const XL_T_STRING: i32 = 0; -pub const XL_T_I64: i32 = 1; -pub const XL_T_F64: i32 = 2; -pub const XL_T_BOOL: i32 = 3; -/// Days since 1970-01-01, stored as `i32`. -pub const XL_T_DATE: i32 = 4; -/// Microseconds since midnight, stored as `i64`. -pub const XL_T_TIME: i32 = 5; -/// Microseconds since 1970-01-01T00:00:00Z, stored as `i64`. -pub const XL_T_TIMESTAMP: i32 = 6; - -/// `xl_infer_schema` flag: also type cells that hold text (integers, decimals, booleans, ISO dates). -pub const XL_INFER_PARSE_TEXT: i32 = 1; - -pub const XL_SHEET_VISIBLE: i32 = 0; -pub const XL_SHEET_HIDDEN: i32 = 1; -pub const XL_SHEET_VERY_HIDDEN: i32 = 2; - -pub const XL_FORMAT_AUTO: i32 = 0; -pub const XL_FORMAT_XLS: i32 = 1; -pub const XL_FORMAT_XLSX: i32 = 2; -pub const XL_FORMAT_XLSB: i32 = 3; -pub const XL_FORMAT_CSV: i32 = 4; - -/// Every boolean-shaped `xl_open_options` field uses one of these three states, never a plain 0/1 - -/// several of them default to true, so a bare 0 would be ambiguous between "off" and "use the -/// library default". -pub const XL_OPT_DEFAULT: i32 = 0; -pub const XL_OPT_FALSE: i32 = 1; -pub const XL_OPT_TRUE: i32 = 2; - -/// Opaque handle - never dereferenced by Rust, only passed back to `xl_*` functions. -#[repr(C)] -pub struct XlWorkbook { - _private: [u8; 0], -} - -/// Opaque row cursor (`xl_rows`) - never dereferenced by Rust, only passed back to `xl_rows_*`. -#[repr(C)] -pub struct XlRows { - _private: [u8; 0], -} - -#[repr(C)] -pub struct XlTypedReader { - _private: [u8; 0], -} - -/// Mirrors `xl_open_options`. `struct_size` must be set to `size_of::()`; build one -/// through [`OpenOptions`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy, Debug)] -pub struct XlOpenOptions { - pub struct_size: i32, - pub csv_sniff_dialect: i32, - pub csv_delimiter: i32, - pub csv_quote: i32, - pub csv_detect_bom: i32, - pub csv_max_cell_bytes: i32, - pub csv_intern_strings: i32, - pub max_total_decompressed_bytes: i64, - pub max_cell_bytes: i32, - pub max_shared_string_bytes: i64, - pub max_zip_entries: i32, - pub prefetch_decompression: i32, - pub intern_strings: i32, - - /// Password for an encrypted OOXML workbook, as UTF-8 bytes. NULL (with `password_len` 0) means - /// "not encrypted, or fail with `XL_STATUS_PASSWORD_REQUIRED`". Not NUL-terminated - - /// `password_len` is authoritative, since a password may contain any byte. The pointer need only - /// remain valid for the duration of the call - see [`crate::options::OpenOptionsRaw`]. - pub password: *const u8, - pub password_len: i32, - pub source_cache_bytes: i64, - pub max_buffered_bytes: i64, - pub source_block_size: i32, - pub verify_zip_checksums: i32, -} - -#[repr(C)] -pub struct XlSource { - pub struct_size: i32, - pub user_data: *mut c_void, - pub length: i64, - pub read_at: Option i64>, - pub release: Option, -} - -#[repr(C)] -pub struct XlStream { - pub struct_size: i32, - pub user_data: *mut c_void, - pub read: Option i64>, - pub release: Option, -} - -/// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with -/// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts -/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. -/// Build one through [`WriteOptions`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy, Debug)] -pub struct XlWriteOptions { - pub struct_size: i32, - pub sheet_name_len: i32, - pub sheet_name: *const u8, - pub csv_delimiter: i32, - pub csv_quote: i32, - pub date1904: i32, - pub use_shared_strings: i32, -} - -#[repr(C)] -pub struct XlColumnSpec { - pub names: *const *const u8, - pub name_lens: *const i32, - pub name_count: i32, - pub index: i32, - pub r#type: i32, - pub nullable: i32, -} - -#[repr(C)] -pub struct XlColumn { - pub r#type: i32, - pub length: i64, - pub values: *const c_void, - pub validity: *const u8, - pub data: *const u8, - pub data_len: i64, -} - -#[repr(C)] -pub struct XlTable { - pub column_count: i32, - pub row_count: i64, - pub columns: *mut XlColumn, -} - -/// Mirrors `xl_inferred_schema`. Owned by the native library until `xl_free_schema`. -#[repr(C)] -pub struct XlInferredSchema { - pub columns: *mut XlColumnSpec, - pub column_count: i32, -} - -/// Mirrors `xl_buffer`: an owned block of native memory returned by `xl_write_typed_to_memory` or -/// `xl_write_handle_bytes`. Released with `xl_free_buffer` - see `writer::buffer_to_vec`, the one -/// place this crate touches the raw struct directly. -#[repr(C)] -pub struct XlBuffer { - pub data: *mut u8, - pub len: i64, -} - -/// Opaque streaming writer handle - never dereferenced by Rust, only passed back to `xl_*` -/// functions. See [`writer_handle::WriterHandle`] for the safe wrapper. -#[repr(C)] -pub struct XlWriterHandle { - _private: [u8; 0], -} - -/// Mirrors `xl_row_cell`. Named `cell_type` because `type` is a Rust keyword. -#[repr(C)] -#[derive(Debug)] -pub struct XlRowCell { - pub column: i32, - pub cell_type: i32, - pub value_len: i32, - pub value: *const u8, -} - -/// Mirrors `xl_row`. -#[repr(C)] -pub struct XlRow { - pub cell_count: i32, - pub cells: *mut XlRowCell, -} - -/// Mirrors `xl_rows_decoded`. -#[repr(C)] -pub struct XlRowsDecoded { - pub row_count: i32, - pub rows: *mut XlRow, -} - -/// Mirrors `xl_csv_aggregation`. Every callback runs on a library worker thread except -/// `free_state`, which runs on the calling thread after the run has finished; none may let a -/// panic escape into the library - build one through [`aggregate`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy)] -pub struct XlCsvAggregation { - pub struct_size: i32, - pub seed: Option i32>, - pub accumulate: Option i32>, - pub combine: Option i32>, - pub free_state: Option, - pub user_data: *mut c_void, -} - -/// Mirrors `xl_csv_parallel_options`. `struct_size` must be set to -/// `size_of::()`; build one through [`CsvParallelOptions`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy, Debug)] -pub struct XlCsvParallelOptions { - pub struct_size: i32, - pub degree_of_parallelism: i32, - pub header_row: i32, - pub delimiter: i32, - pub quote: i32, - pub detect_bom: i32, - pub max_cell_bytes: i32, -} - -extern "C" { - pub fn xl_abi_version() -> c_int; - - pub fn xl_open_file( - path: *const u8, - path_len: i32, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_open_memory( - data: *const u8, - data_len: i32, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_open_source( - source: *const XlSource, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_open_stream( - stream: *const XlStream, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_set_source_error(message: *const u8, len: i32); - - pub fn xl_close(handle: *mut XlWorkbook) -> c_int; - - pub fn xl_sheet_count(handle: *mut XlWorkbook, out_count: *mut i32) -> c_int; - - pub fn xl_sheet_name_at( - handle: *mut XlWorkbook, - index: i32, - buffer: *mut u8, - capacity: i32, - out_len: *mut i32, - ) -> c_int; - - pub fn xl_sheet_visibility_at( - handle: *mut XlWorkbook, - index: i32, - out_visibility: *mut i32, - ) -> c_int; - - /// A name that matches no sheet is `XL_OK` with `-1`. - pub fn xl_sheet_index( - handle: *mut XlWorkbook, - name: *const u8, - name_len: i32, - out_index: *mut i32, - ) -> c_int; - - pub fn xl_is_date1904(handle: *mut XlWorkbook, out_flag: *mut i32) -> c_int; - - pub fn xl_parse_typed( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - degree_of_parallelism: i32, - out_table: *mut XlTable, - ) -> c_int; - - /// Same schema-driven parse as `xl_parse_typed`, exported as one top-level Arrow struct - /// array/schema. `out_array`/`out_schema` are `struct ArrowArray*`/`struct ArrowSchema*` from - /// the Arrow C Data Interface, typed here as `c_void` because arrow-rs's own `#[repr(C)]` - /// `FFI_ArrowArray`/`FFI_ArrowSchema` are ABI-identical to them - redeclaring the spec structs - /// would be a second source of truth for a fixed, versioned ABI. - /// - /// On `XL_OK` the caller owns both and releases each through its OWN `release` callback, never - /// through `xl_free_table`. On any other status both outputs are left untouched. - pub fn xl_parse_arrow( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - degree_of_parallelism: i32, - out_array: *mut c_void, - out_schema: *mut c_void, - ) -> c_int; - - /// `out_stream` is a `struct ArrowArrayStream*`, typed as `c_void` for the same reason - /// `xl_parse_arrow`'s outputs are. Its `release` is what closes the underlying read. - pub fn xl_parse_arrow_stream( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - max_rows: i64, - out_stream: *mut c_void, - ) -> c_int; - - pub fn xl_free_table(table: *mut XlTable); - - /// `max_rows` is rows per batch, 0 meaning one unbounded batch. - pub fn xl_typed_reader_open( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - max_rows: i64, - out_reader: *mut *mut XlTypedReader, - ) -> c_int; - - /// `XL_OK` with a batch, or `XL_EOF` at end of sheet. A failure latches: every later call - /// repeats it. - pub fn xl_typed_reader_next(reader: *mut XlTypedReader, out_table: *mut XlTable) -> c_int; - - pub fn xl_typed_reader_close(reader: *mut XlTypedReader); - - pub fn xl_infer_schema( - handle: *mut XlWorkbook, - sheet: i32, - header_row: i32, - sample_size: i32, - flags: i32, - out_schema: *mut XlInferredSchema, - ) -> c_int; - - pub fn xl_free_schema(schema: *mut XlInferredSchema); - pub fn xl_write_typed( - path: *const u8, - path_len: i32, - format: i32, - specs: *const XlColumnSpec, - table: *const XlTable, - options: *const XlWriteOptions, - ) -> c_int; - - /// Same as `xl_write_typed`, except the result is returned as `out_buffer` instead of being - /// written to a path. Only read `*out_buffer` when the call returns `XL_OK`; on failure it is - /// zeroed, and `xl_free_buffer` on a zeroed buffer is a no-op. - pub fn xl_write_typed_to_memory( - format: i32, - specs: *const XlColumnSpec, - table: *const XlTable, - options: *const XlWriteOptions, - out_buffer: *mut XlBuffer, - ) -> c_int; - - /// Releases a buffer returned by `xl_write_typed_to_memory` or `xl_write_handle_bytes` and - /// resets it to zero. Safe on a zeroed value. - pub fn xl_free_buffer(buffer: *mut XlBuffer); - - /// Reads the plaintext XLSX/XLSB package at `package_path` and writes its agile-encrypted - /// (ECMA-376 4.4) counterpart to `destination_path`, overwriting an existing file. The result - /// opens with the same password via `xl_open_file`'s `xl_open_options::password`. - pub fn xl_encrypt_package( - package_path: *const u8, - package_path_len: i32, - destination_path: *const u8, - destination_path_len: i32, - password: *const u8, - password_len: i32, - ) -> c_int; - - - pub fn xl_open_write_handle( - path: *const u8, - path_len: i32, - format: i32, - options: *const XlWriteOptions, - out_handle: *mut *mut XlWriterHandle, - ) -> c_int; - - pub fn xl_open_write_handle_to_memory( - format: i32, - options: *const XlWriteOptions, - out_handle: *mut *mut XlWriterHandle, - ) -> c_int; - - pub fn xl_start_sheet(handle: *mut XlWriterHandle, name: *const u8, name_len: i32) -> c_int; - pub fn xl_start_row(handle: *mut XlWriterHandle) -> c_int; - - pub fn xl_write_string(handle: *mut XlWriterHandle, value: *const u8, value_len: i32) -> c_int; - pub fn xl_write_int64(handle: *mut XlWriterHandle, value: i64) -> c_int; - pub fn xl_write_float64(handle: *mut XlWriterHandle, value: f64) -> c_int; - pub fn xl_write_bool(handle: *mut XlWriterHandle, value: i32) -> c_int; - pub fn xl_write_date(handle: *mut XlWriterHandle, days_since_epoch: i32) -> c_int; - pub fn xl_write_time(handle: *mut XlWriterHandle, micros_since_midnight: i64) -> c_int; - pub fn xl_write_timestamp(handle: *mut XlWriterHandle, micros_since_epoch: i64) -> c_int; - pub fn xl_write_null(handle: *mut XlWriterHandle, r#type: i32) -> c_int; - - pub fn xl_end_row(handle: *mut XlWriterHandle) -> c_int; - pub fn xl_end_sheet(handle: *mut XlWriterHandle) -> c_int; - pub fn xl_close_write_handle(handle: *mut XlWriterHandle) -> c_int; - - /// Reads back everything written so far to a handle opened by `xl_open_write_handle_to_memory`. - /// `XL_INVALID_ARGUMENT` for one from `xl_open_write_handle`. Only read `*out_buffer` when the - /// call returns `XL_OK`; on failure it is zeroed, and `xl_free_buffer` on a zeroed buffer is a - /// no-op. - pub fn xl_write_handle_bytes(handle: *mut XlWriterHandle, out_buffer: *mut XlBuffer) -> c_int; - - pub fn xl_rows_open(handle: *mut XlWorkbook, sheet: i32, out_rows: *mut *mut XlRows) -> c_int; - pub fn xl_rows_close(rows: *mut XlRows) -> c_int; - pub fn xl_rows_next( - rows: *mut XlRows, - buffer: *mut u8, - capacity: i32, - out_written: *mut i32, - ) -> c_int; - pub fn xl_rows_read_all_blob( - rows: *mut XlRows, - buffer: *mut u8, - capacity: i32, - out_written: *mut i32, - ) -> c_int; - pub fn xl_rows_read_all_decoded(rows: *mut XlRows, out_rows: *mut XlRowsDecoded) -> c_int; - pub fn xl_free_rows(rows: *mut XlRowsDecoded); - - /// Folds a CSV file into one caller-owned accumulator across several threads. Blocks until the - /// run finishes; the surviving state is written to `out_state` only on `XL_OK` and becomes the - /// caller's to free. See the contract above `xl_csv_aggregation` in `excelreader.h`, and - /// [`aggregate::aggregate_csv_file`] for the safe wrapper. - pub fn xl_csv_aggregate_file( - path: *const u8, - path_len: i32, - agg: *const XlCsvAggregation, - options: *const XlCsvParallelOptions, - out_state: *mut *mut c_void, - ) -> c_int; - - /// Same as `xl_csv_aggregate_file` over an in-memory buffer, which must stay valid and - /// unmodified until the call returns - the library does not copy it. - pub fn xl_csv_aggregate_memory( - data: *const u8, - data_len: i32, - agg: *const XlCsvAggregation, - options: *const XlCsvParallelOptions, - out_state: *mut *mut c_void, - ) -> c_int; - - pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; +#[cfg(feature = "arrow")] +pub use self::arrow::ArrowBatches; + +/// Support for `#[derive(ExcelMapper)]`'s generated code. Not part of the public API: nothing here +/// follows semver. +#[doc(hidden)] +pub mod __private { + pub use crate::ffi::XlColumn; + pub use crate::workbook::{ + column_bool, column_date, column_f64, column_i64, column_str, column_time, + column_timestamp, ColumnBinding, + }; + pub use crate::writer::{push_str, set_valid, OwnedColumn, OwnedColumnData}; + + #[must_use] + pub fn invalid_argument(message: String) -> crate::Error { + crate::Error::new(crate::ErrorKind::InvalidArgument, message) + } } diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 8f9342ec..417b99a0 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -1,56 +1,103 @@ -//! Builder for [`XlOpenOptions`], mirroring the Python binding's `OpenOptions` and the C++ -//! wrapper's `xl::OpenOptions`. +//! Builders for the ABI's open, write and CSV-aggregation option structs, mirroring the Python +//! binding's `OpenOptions` and the C++ wrapper's `xl::OpenOptions`. //! //! The ABI splits "unset" across two conventions: `0` for a numeric field, `XL_OPT_DEFAULT` for a -//! boolean-shaped one. This type presents the single `None` convention the other bindings expose and -//! does the split in [`OpenOptions::to_raw`]. +//! boolean-shaped one. These types present the single `None` convention the other bindings expose +//! and do the split when lowering. //! //! Values are passed through unvalidated on purpose: the native side owns the real bounds and //! reports a rejection through `xl_last_error`, so checking them here too would give those bounds a //! second place to drift from. -use crate::{XlOpenOptions, XlWriteOptions, XL_OPT_DEFAULT, XL_OPT_FALSE, XL_OPT_TRUE}; +use std::path::Path; + +use crate::ffi::{ + self, XlCsvParallelOptions, XlOpenOptions, XlWriteOptions, XL_OPT_DEFAULT, XL_OPT_FALSE, + XL_OPT_TRUE, +}; +use crate::{Error, ErrorKind}; + +/// A workbook file format. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Format { + /// Opening: sniff the format from the file's signature bytes. Sniffing does not detect CSV. + /// Keep `Auto` for a password-protected workbook: an explicit `Xlsx`/`Xlsb` reads its + /// encryption wrapper as a plain ZIP and fails. Writing: infer the format from the path's + /// extension. + #[default] + Auto, + Xls, + Xlsx, + Xlsb, + Csv, +} + +impl Format { + pub(crate) fn raw(self) -> i32 { + match self { + Format::Auto => ffi::XL_FORMAT_AUTO, + Format::Xls => ffi::XL_FORMAT_XLS, + Format::Xlsx => ffi::XL_FORMAT_XLSX, + Format::Xlsb => ffi::XL_FORMAT_XLSB, + Format::Csv => ffi::XL_FORMAT_CSV, + } + } -/// Options for [`Workbook::open_with`](crate::workbook::Workbook::open_with) and -/// [`Workbook::open_memory`](crate::workbook::Workbook::open_memory). Every field is `None` by -/// default, meaning "use the library default"; set only the ones you want to override. + /// Infers a format from a path's extension, case-insensitively. `Auto` when the extension is + /// absent or unrecognized - which the native write then rejects, since a file being created has + /// no signature bytes to sniff. + fn from_path(path: &Path) -> Format { + let extension = path + .extension() + .and_then(|e| e.to_str()) + .unwrap_or_default(); + match extension.to_ascii_lowercase().as_str() { + "xlsx" => Format::Xlsx, + "xlsb" => Format::Xlsb, + "xls" => Format::Xls, + "csv" => Format::Csv, + _ => Format::Auto, + } + } +} + +/// How to open a workbook, in the style of [`std::fs::OpenOptions`]: build one, then call +/// [`open`](Self::open), [`open_bytes`](Self::open_bytes), [`open_source`](Self::open_source) or +/// [`open_reader`](Self::open_reader). Every setting is unset by default, meaning "use the library +/// default"; set only the ones you want to override. /// /// ```no_run -/// use excelreader::OpenOptions; +/// use excelreader::{Format, OpenOptions}; /// -/// let options = OpenOptions::new() -/// .prefetch_decompression(true) -/// .max_zip_entries(1024); +/// let workbook = OpenOptions::new() +/// .format(Format::Csv) +/// .csv_delimiter(b';' as i32) +/// .open("data.csv")?; +/// # Ok::<(), excelreader::Error>(()) /// ``` #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct OpenOptions { - pub csv_sniff_dialect: Option, - pub csv_delimiter: Option, - pub csv_quote: Option, - pub csv_detect_bom: Option, - pub csv_max_cell_bytes: Option, - pub csv_intern_strings: Option, - - pub max_total_decompressed_bytes: Option, - pub max_cell_bytes: Option, - pub max_shared_string_bytes: Option, - pub max_zip_entries: Option, - pub prefetch_decompression: Option, - pub intern_strings: Option, - /// XLSX/XLSB only: check each ZIP entry against its CRC-32. `None` is `true`. - pub verify_zip_checksums: Option, - - /// `open_source` only: bytes fetched per `read_at` call. `None` is 4 MiB; a negative value turns - /// the cache off. - pub source_block_size: Option, - /// `open_source` only: bytes of blocks kept in memory. `None` is 64 MiB. - pub source_cache_bytes: Option, - /// `open_reader` with XLSX, XLSB or XLS only: the most bytes the reader may hold. - pub max_buffered_bytes: Option, - - /// Password for an encrypted OOXML workbook. `None` means "not encrypted, or fail with - /// `Error::PasswordRequired`". - pub password: Option, + format: Format, + + csv_sniff_dialect: Option, + csv_delimiter: Option, + csv_quote: Option, + csv_detect_bom: Option, + + max_total_decompressed_bytes: Option, + max_cell_bytes: Option, + max_shared_string_bytes: Option, + max_zip_entries: Option, + prefetch_decompression: Option, + intern_strings: Option, + verify_zip_checksums: Option, + + source_block_size: Option, + source_cache_bytes: Option, + max_buffered_bytes: Option, + + password: Option, } /// Generates a consuming builder setter per field, so callers can chain overrides. @@ -68,12 +115,20 @@ macro_rules! setters { } impl OpenOptions { - /// Every field unset - identical to passing no options at all. + /// Every setting unset: sniff the format, use every library default. #[must_use] pub fn new() -> Self { Self::default() } + /// The file format. Default [`Format::Auto`], which does not detect CSV - set + /// [`Format::Csv`] to open one. + #[must_use] + pub fn format(mut self, value: Format) -> Self { + self.format = value; + self + } + setters! { /// CSV only. Infers delimiter/quote/encoding from a leading sample before opening, which /// makes `csv_delimiter`/`csv_quote` ignored. @@ -84,13 +139,9 @@ impl OpenOptions { csv_quote: i32, /// CSV only. Library default is `true`. csv_detect_bom: bool, - /// CSV only. Library default is 32 MiB. - csv_max_cell_bytes: i32, - /// CSV only. Library default is `false`. - csv_intern_strings: bool, /// XLS/XLSX/XLSB only. Library default is 512 MiB. max_total_decompressed_bytes: i64, - /// XLS/XLSX/XLSB only. Library default is 32 MiB. + /// Largest cell value, or buffered CSV record. Library default is 32 MiB. max_cell_bytes: i32, /// XLS/XLSX/XLSB only. Library default is 128 MiB. max_shared_string_bytes: i64, @@ -98,41 +149,45 @@ impl OpenOptions { max_zip_entries: i32, /// XLS/XLSX/XLSB only. Library default is `false`. prefetch_decompression: bool, - /// XLS/XLSX/XLSB only. Library default is `false`. + /// Deduplicate repeated cell strings. Library default is `false`. intern_strings: bool, /// XLSX/XLSB only: check each ZIP entry against its CRC-32, so a damaged file fails the read /// instead of returning wrong cells. Library default is `true`. verify_zip_checksums: bool, - /// `open_source` only: bytes fetched per `read_at` call. Library default is 4 MiB; a negative - /// value turns the cache off. + /// [`open_source`](Self::open_source) only: bytes fetched per `read_at` call. Library + /// default is 4 MiB; a negative value turns the cache off. source_block_size: i32, - /// `open_source` only: bytes of blocks kept in memory. Library default is 64 MiB. + /// [`open_source`](Self::open_source) only: bytes of blocks kept in memory. Library + /// default is 64 MiB. source_cache_bytes: i64, - /// `open_reader` with XLSX, XLSB or XLS only: the most bytes the reader may hold. + /// [`open_reader`](Self::open_reader) with XLSX, XLSB or XLS only: the most bytes the + /// reader may hold. max_buffered_bytes: i64, } - /// Password for an encrypted OOXML workbook. Stored owned, since the C ABI receives a pointer - /// that must stay valid for the duration of the open call. - /// - /// Written by hand rather than generated by `setters!`: this is the one field that is not a - /// plain `Copy` scalar. + /// Password for an encrypted OOXML workbook. Unset means "not encrypted, or fail with + /// [`ErrorKind::PasswordRequired`]". #[must_use] pub fn password(mut self, value: impl Into) -> Self { self.password = Some(value.into()); self } + pub(crate) fn format_raw(&self) -> i32 { + self.format.raw() + } + /// Lowers this into the raw ABI struct, with `struct_size` filled in. /// /// Returns a guard rather than a bare `XlOpenOptions`: when [`password`](Self::password) is /// set, the raw struct's `password` field is a pointer into `self.password`'s bytes, which must - /// outlive every use of the pointer. Returning `XlOpenOptions` by value would let that pointer - /// dangle the moment the return value's producer (this call) went out of scope - the borrow on - /// [`OpenOptionsRaw`] below is what makes the compiler refuse a call site that does not keep - /// `self` alive for as long as the raw struct is in use. - #[must_use] - pub fn to_raw(&self) -> OpenOptionsRaw<'_> { + /// outlive every use of the pointer. The borrow on [`OpenOptionsRaw`] is what makes the + /// compiler refuse a call site that does not keep `self` alive for as long as the raw struct is + /// in use. + /// + /// `max_cell_bytes`/`intern_strings` lower into both the general and the `csv_*` fields: the + /// ABI reads the `csv_*` pair for CSV and the other pair for every other format. + pub(crate) fn to_raw(&self) -> OpenOptionsRaw<'_> { let password = self.password.as_deref().map(str::as_bytes); let mut raw = XlOpenOptions { struct_size: std::mem::size_of::() as i32, @@ -140,8 +195,8 @@ impl OpenOptions { csv_delimiter: opt_number(self.csv_delimiter), csv_quote: opt_number(self.csv_quote), csv_detect_bom: opt_state(self.csv_detect_bom), - csv_max_cell_bytes: opt_number(self.csv_max_cell_bytes), - csv_intern_strings: opt_state(self.csv_intern_strings), + csv_max_cell_bytes: opt_number(self.max_cell_bytes), + csv_intern_strings: opt_state(self.intern_strings), max_total_decompressed_bytes: opt_number(self.max_total_decompressed_bytes), max_cell_bytes: opt_number(self.max_cell_bytes), max_shared_string_bytes: opt_number(self.max_shared_string_bytes), @@ -164,25 +219,36 @@ impl OpenOptions { _password: password, } } + + /// The CSV dialect and cell limit of these options, plus the two per-call settings, as the + /// aggregation ABI takes them. + pub(crate) fn to_csv_parallel_raw( + &self, + header_row: i32, + degree_of_parallelism: i32, + ) -> XlCsvParallelOptions { + XlCsvParallelOptions { + struct_size: std::mem::size_of::() as i32, + degree_of_parallelism, + header_row, + delimiter: opt_number(self.csv_delimiter), + quote: opt_number(self.csv_quote), + detect_bom: opt_state(self.csv_detect_bom), + max_cell_bytes: opt_number(self.max_cell_bytes), + } + } } /// Holds the raw C struct together with the buffer its `password` pointer refers to. The raw struct /// must never outlive this guard, and the borrow of `OpenOptions` (carried through `'a`) is what -/// guarantees it: `_password` borrows the same `OpenOptions::password` string `raw.password` points -/// into, so the borrow checker keeps this guard's lifetime from exceeding `self`'s. -/// -/// `_password` is never read - it exists purely to hold that borrow. Dropping it early would not -/// itself free anything (the bytes live in `OpenOptions`, not here), but nothing stops a future edit -/// from reordering fields in a way that matters, so keep it paired with `raw` for the same reason -/// `SpecArena` in `workbook.rs` keeps its own otherwise-unread borrow fields. -pub struct OpenOptionsRaw<'a> { +/// guarantees it. +pub(crate) struct OpenOptionsRaw<'a> { raw: XlOpenOptions, _password: Option<&'a [u8]>, } impl OpenOptionsRaw<'_> { - /// Pointer to the raw struct, valid for as long as this guard is alive (and no longer - the - /// password bytes it may point into are borrowed from the `OpenOptions` this guard came from). + /// Pointer to the raw struct, valid for as long as this guard is alive. pub(crate) fn as_ptr(&self) -> *const XlOpenOptions { &self.raw } @@ -203,8 +269,65 @@ fn opt_number(value: Option) -> T { value.unwrap_or_default() } -/// Options for [`write_columns`](crate::writer::write_columns) and -/// [`write_sheet`](crate::writer::write_sheet). Every field is `None` by default, meaning "use the +/// How a sheet is read by [`Sheet::parse`](crate::Sheet::parse), +/// [`Sheet::parse_batches`](crate::Sheet::parse_batches) and their Arrow counterparts. +/// +/// ```no_run +/// use excelreader::ReadOptions; +/// +/// let options = ReadOptions::new().header_row(2).parallelism(0); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReadOptions { + pub(crate) header_row: i32, + pub(crate) parallelism: i32, + pub(crate) batch_size: i64, +} + +impl Default for ReadOptions { + fn default() -> Self { + Self { + header_row: 1, + parallelism: 1, + batch_size: 10_000, + } + } +} + +impl ReadOptions { + /// Header in row 1, one thread, 10,000 rows per batch. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// 1-based row whose values name the columns; `0` means no header, and columns are bound by + /// position. + #[must_use] + pub fn header_row(mut self, value: i32) -> Self { + self.header_row = value; + self + } + + /// Threads for a whole-sheet read: `0` every processor, `1` sequential, `n` up to `n`. The + /// result is the same whatever it is. Batched reads ignore it. + #[must_use] + pub fn parallelism(mut self, value: i32) -> Self { + self.parallelism = value; + self + } + + /// Rows per batch for a batched read, `0` meaning one unbounded batch. Whole-sheet reads + /// ignore it. + #[must_use] + pub fn batch_size(mut self, value: i64) -> Self { + self.batch_size = value; + self + } +} + +/// Options for [`write_columns`](crate::write_columns), [`write_sheet`](crate::write_sheet) and +/// [`WriterHandle`](crate::WriterHandle). Every setting is unset by default, meaning "use the /// library default"; set only the ones you want to override. /// /// ```no_run @@ -214,29 +337,31 @@ fn opt_number(value: Option) -> T { /// ``` #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct WriteOptions { - /// Sheet name. `None` = `"Sheet1"`. Ignored for CSV. Excel's rules (1-31 characters, none of - /// `: \ / ? * [ ]`) are enforced natively and reported through `xl_last_error`. - pub sheet_name: Option, - - pub csv_delimiter: Option, - pub csv_quote: Option, - - /// XLS/XLSB only; ignored for XLSX and CSV. Library default is `false`. - pub date1904: Option, - /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at the - /// cost of a string table. Library default is `false`. - pub use_shared_strings: Option, + format: Format, + sheet_name: Option, + csv_delimiter: Option, + csv_quote: Option, + date1904: Option, + use_shared_strings: Option, } impl WriteOptions { - /// Every field unset - identical to passing no options at all. + /// Every setting unset. #[must_use] pub fn new() -> Self { Self::default() } - /// Written by hand rather than generated by `setters!`: this is the one field that is not a - /// plain `Copy` scalar. + /// The output format. Default [`Format::Auto`]: inferred from the path's extension + /// (`.xlsx`/`.xlsb`/`.xls`/`.csv`). Writing to an `impl Write` needs an explicit format. + #[must_use] + pub fn format(mut self, value: Format) -> Self { + self.format = value; + self + } + + /// Sheet name. Unset is `"Sheet1"`. Ignored for CSV. Excel's rules (1-31 characters, none of + /// `: \ / ? * [ ]`) are enforced natively. #[must_use] pub fn sheet_name(mut self, value: impl Into) -> Self { self.sheet_name = Some(value.into()); @@ -248,18 +373,28 @@ impl WriteOptions { csv_delimiter: i32, /// CSV only. Byte value 1-255; the library default is `"`. csv_quote: i32, - /// XLS/XLSB only. Library default is `false`. + /// XLS/XLSB only; ignored for XLSX and CSV. Library default is `false`. date1904: bool, - /// XLSX/XLSB only. Library default is `false`. + /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at + /// the cost of a string table. Library default is `false`. use_shared_strings: bool, } - /// Lowers this into the raw ABI struct, with `struct_size` filled in. - /// - /// `pub(crate)`, unlike [`OpenOptions::to_raw`], which is public: the struct this returns holds - /// a raw pointer INTO `self.sheet_name`, so handing it to a caller who can outlive `self` is a - /// use-after-free waiting to happen. `write_columns` takes `Option<&WriteOptions>` and does the - /// lowering itself, where the borrow provably covers the FFI call. + /// The `XL_FORMAT_*` to write: the explicit format, else one inferred from `path`. With no path + /// to infer from, `Auto` is rejected here rather than by the native side. + pub(crate) fn format_for(&self, path: Option<&Path>) -> Result { + match (self.format, path) { + (Format::Auto, Some(path)) => Ok(Format::from_path(path).raw()), + (Format::Auto, None) => Err(Error::new( + ErrorKind::InvalidArgument, + "writing to an io::Write needs an explicit WriteOptions::format", + )), + (format, _) => Ok(format.raw()), + } + } + + /// Lowers this into the raw ABI struct, with `struct_size` filled in. The struct holds a raw + /// pointer INTO `self.sheet_name`, so it must not outlive `self`. pub(crate) fn to_raw(&self) -> XlWriteOptions { XlWriteOptions { struct_size: std::mem::size_of::() as i32, @@ -276,15 +411,6 @@ impl WriteOptions { } } -/// The pointer an FFI options argument takes: a borrowed raw struct, or NULL for "every library -/// default". `raw` must outlive the call the returned pointer is handed to - taking it by reference -/// rather than by value is what makes the compiler check that. -#[inline] -pub(crate) fn ptr_or_null(raw: &Option) -> *const T { - raw.as_ref() - .map_or(std::ptr::null(), |value| value as *const T) -} - #[cfg(test)] mod tests { use super::*; @@ -304,6 +430,7 @@ mod tests { "no password set means a null pointer" ); assert_eq!(raw.password_len, 0); + assert_eq!(options.format_raw(), ffi::XL_FORMAT_AUTO); } #[test] @@ -333,6 +460,36 @@ mod tests { assert_eq!(raw.max_shared_string_bytes, 1 << 20); } + #[test] + fn cell_limit_and_interning_reach_both_the_csv_and_the_general_fields() { + let options = OpenOptions::new().max_cell_bytes(4096).intern_strings(true); + let guard = options.to_raw(); + let raw = guard.raw; + assert_eq!(raw.max_cell_bytes, 4096); + assert_eq!(raw.csv_max_cell_bytes, 4096); + assert_eq!(raw.intern_strings, XL_OPT_TRUE); + assert_eq!(raw.csv_intern_strings, XL_OPT_TRUE); + } + + #[test] + fn csv_aggregation_options_take_the_dialect_and_cell_limit() { + let raw = OpenOptions::new() + .csv_delimiter(b';' as i32) + .csv_detect_bom(false) + .max_cell_bytes(1024) + .to_csv_parallel_raw(1, 4); + assert_eq!( + raw.struct_size, + std::mem::size_of::() as i32 + ); + assert_eq!(raw.header_row, 1); + assert_eq!(raw.degree_of_parallelism, 4); + assert_eq!(raw.delimiter, b';' as i32); + assert_eq!(raw.quote, 0); + assert_eq!(raw.detect_bom, XL_OPT_FALSE); + assert_eq!(raw.max_cell_bytes, 1024); + } + #[test] fn password_lowers_to_a_pointer_and_length_into_the_owning_options() { let options = OpenOptions::new().password("hunter2"); @@ -344,11 +501,8 @@ mod tests { assert_eq!(bytes, b"hunter2"); } - /// The regression this task exists to prevent: `to_raw` must not compile down to something that - /// lets the password pointer dangle when the source string is a temporary rather than a local - /// the caller happens to keep alive. `to_raw`'s borrow of `options` (carried by `OpenOptionsRaw`) - /// is what makes that impossible - `options` here is a local binding that outlives `guard`, even - /// though the `String` handed to `.password()` itself was a temporary. + /// `to_raw` must not compile down to something that lets the password pointer dangle when the + /// source string is a temporary rather than a local the caller happens to keep alive. #[test] fn password_survives_a_temporary_source() { let options = OpenOptions::new().password(String::from("hunter2")); @@ -363,7 +517,7 @@ mod tests { let raw = WriteOptions::new().to_raw(); assert_eq!( raw.struct_size, - std::mem::size_of::() as i32 + std::mem::size_of::() as i32 ); assert!(raw.sheet_name.is_null(), "an unset sheet name means Sheet1"); assert_eq!(raw.sheet_name_len, 0); @@ -386,22 +540,50 @@ mod tests { assert_eq!(raw.use_shared_strings, XL_OPT_TRUE); } + #[test] + fn write_format_is_explicit_or_inferred_from_the_extension() { + let auto = WriteOptions::new(); + let format = |path: &str| auto.format_for(Some(Path::new(path))).unwrap(); + assert_eq!(format("out.xlsx"), ffi::XL_FORMAT_XLSX); + assert_eq!(format("out.XLSB"), ffi::XL_FORMAT_XLSB); + assert_eq!(format("out.xls"), ffi::XL_FORMAT_XLS); + assert_eq!(format("out.csv"), ffi::XL_FORMAT_CSV); + assert_eq!(format("out.txt"), ffi::XL_FORMAT_AUTO); + assert_eq!(format("out"), ffi::XL_FORMAT_AUTO); + assert_eq!(format("v1.2/report"), ffi::XL_FORMAT_AUTO); + + let error = auto + .format_for(None) + .expect_err("a writer has no extension to infer from"); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + + let csv = WriteOptions::new().format(Format::Csv); + assert_eq!( + csv.format_for(Some(Path::new("out.xlsx"))).unwrap(), + ffi::XL_FORMAT_CSV + ); + assert_eq!(csv.format_for(None).unwrap(), ffi::XL_FORMAT_CSV); + } + /// The C struct's x64 layout is pinned by XL_STATIC_ASSERT in - /// tests/ExcelReader.NativeSmoke/smoke.c (lines 66-73). A repr(C) struct that disagrees would - /// hand the native side garbage where it expects a length and a pointer. + /// tests/ExcelReader.NativeSmoke/smoke.c. A repr(C) struct that disagrees would hand the native + /// side garbage where it expects a length and a pointer. #[test] fn write_options_match_the_c_struct_size() { - assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 32); } #[test] fn source_layouts_match_the_c_structs() { - use crate::{XlOpenOptions, XlSource, XlStream}; + use crate::ffi::{XlSource, XlStream}; assert_eq!(std::mem::size_of::(), 112); assert_eq!(std::mem::offset_of!(XlOpenOptions, source_cache_bytes), 88); assert_eq!(std::mem::offset_of!(XlOpenOptions, max_buffered_bytes), 96); assert_eq!(std::mem::offset_of!(XlOpenOptions, source_block_size), 104); - assert_eq!(std::mem::offset_of!(XlOpenOptions, verify_zip_checksums), 108); + assert_eq!( + std::mem::offset_of!(XlOpenOptions, verify_zip_checksums), + 108 + ); assert_eq!(std::mem::size_of::(), 40); assert_eq!(std::mem::size_of::(), 32); } diff --git a/rust/excelreader/src/rows.rs b/rust/excelreader/src/rows.rs index c516de9a..c075528f 100644 --- a/rust/excelreader/src/rows.rs +++ b/rust/excelreader/src/rows.rs @@ -1,13 +1,15 @@ //! Row-at-a-time and whole-sheet decoded reads over the C ABI's row APIs. -use crate::{ - Error, XlRowCell, XL_CELL_BOOL, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR, XL_CELL_FORMULA, - XL_CELL_NUMBER, XL_CELL_STRING, XL_ERROR, +use crate::ffi::{ + self, XlRowCell, XL_CELL_BOOL, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR, XL_CELL_FORMULA, + XL_CELL_NUMBER, XL_CELL_STRING, }; +use crate::{Error, ErrorKind}; /// The kind of a cell, mirroring `XL_CELL_*` in the C ABI. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(i32)] +#[non_exhaustive] pub enum CellType { Empty = XL_CELL_EMPTY, String = XL_CELL_STRING, @@ -21,8 +23,7 @@ pub enum CellType { impl CellType { /// Maps a raw `XL_CELL_*` value. `None` for a value this crate does not know, which can only /// happen against a newer native library than the one this crate was built against. - #[must_use] - pub fn from_raw(value: i32) -> Option { + pub(crate) fn from_raw(value: i32) -> Option { match value { XL_CELL_EMPTY => Some(CellType::Empty), XL_CELL_STRING => Some(CellType::String), @@ -54,13 +55,17 @@ impl<'a> CellRef<'a> { /// The value as a string. Fails when the bytes are not valid UTF-8, which a well-formed /// workbook never produces. pub fn as_str(&self) -> Result<&'a str, Error> { - std::str::from_utf8(self.value) - .map_err(|err| Error::from_status(XL_ERROR, format!("cell value is not valid UTF-8: {err}"))) + std::str::from_utf8(self.value).map_err(|err| { + Error::new( + ErrorKind::Other, + format!("cell value is not valid UTF-8: {err}"), + ) + }) } } -/// Where a `RowRef`'s cells live. Blob rows come from `xl_rows_next`, decoded rows from -/// `xl_rows_read_all_decoded`. +/// Where a `RowRef`'s cells live. Blob rows come from `xl_rows_next`/`xl_rows_read_all_blob`, +/// decoded rows from a CSV aggregation callback. #[derive(Debug, Clone, Copy)] enum RowBacking<'a> { /// The bytes AFTER the leading `int32 cell_count`. @@ -93,7 +98,7 @@ impl<'a> RowRef<'a> { /// /// # Safety /// `cells` must point to `count` initialized `XlRowCell` values whose `value` pointers stay - /// valid for `'a` — that is, until `xl_free_rows` releases the enclosing `XlRowsDecoded`. + /// valid for `'a`. pub(crate) unsafe fn from_decoded(cells: *const XlRowCell, count: i32) -> RowRef<'a> { let len = if count > 0 { count as usize } else { 0 }; let slice = if len == 0 || cells.is_null() { @@ -101,7 +106,10 @@ impl<'a> RowRef<'a> { } else { unsafe { std::slice::from_raw_parts(cells, len) } }; - RowRef { backing: RowBacking::Decoded(slice), len } + RowRef { + backing: RowBacking::Decoded(slice), + len, + } } #[must_use] @@ -114,9 +122,8 @@ impl<'a> RowRef<'a> { self.len == 0 } - /// The cell at `index`. For a row from `RowCursor` this walks the blob from the start, so it is - /// O(index); prefer `iter()` when reading a whole row. A row from `DecodedRows` indexes - /// directly. + /// The cell at `index`. For a row read from a sheet this walks the row from the start, so it is + /// O(index); prefer `iter()` when reading a whole row. #[must_use] pub fn get(&self, index: usize) -> Option> { if index >= self.len { @@ -130,7 +137,12 @@ impl<'a> RowRef<'a> { #[must_use] pub fn iter(&self) -> CellIter<'a> { - CellIter { backing: self.backing, len: self.len, index: 0, offset: 0 } + CellIter { + backing: self.backing, + len: self.len, + index: 0, + offset: 0, + } } } @@ -200,7 +212,14 @@ fn cell_from_blob(blob: &[u8], offset: usize) -> Option<(CellRef<'_>, usize)> { let end = start.checked_add(value_len as usize)?; let value = blob.get(start..end)?; let cell_type = CellType::from_raw(raw_type)?; - Some((CellRef { column, cell_type, value }, end)) + Some(( + CellRef { + column, + cell_type, + value, + }, + end, + )) } fn cell_from_decoded(raw: &XlRowCell) -> Option> { @@ -210,7 +229,11 @@ fn cell_from_decoded(raw: &XlRowCell) -> Option> { } else { unsafe { std::slice::from_raw_parts(raw.value, raw.value_len as usize) } }; - Some(CellRef { column: raw.column, cell_type, value }) + Some(CellRef { + column: raw.column, + cell_type, + value, + }) } /// Rows are usually well under this; it only sets how often the first oversized row costs a retry. @@ -222,19 +245,19 @@ const INITIAL_ROW_BUFFER: usize = 64 * 1024; /// Not an `Iterator`: each row borrows the buffer that the next call overwrites, which /// `Iterator::next` cannot express. One row is alive at a time, enforced by the borrow checker. pub struct RowCursor<'w> { - rows: *mut crate::XlRows, + rows: *mut ffi::XlRows, buffer: Vec, written: usize, - workbook: std::marker::PhantomData<&'w crate::workbook::Workbook>, + workbook: std::marker::PhantomData<&'w crate::Workbook>, } // SAFETY: the cursor owns its native enumerator and ABI v6 lets one thread at a time use it; unsafe impl Send for RowCursor<'_> {} impl<'w> RowCursor<'w> { - pub(crate) fn open(handle: *mut crate::XlWorkbook, sheet: i32) -> Result, Error> { - let mut rows: *mut crate::XlRows = std::ptr::null_mut(); - crate::workbook::check(unsafe { crate::xl_rows_open(handle, sheet, &mut rows) })?; + pub(crate) fn open(handle: *mut ffi::XlWorkbook, sheet: i32) -> Result, Error> { + let mut rows: *mut ffi::XlRows = std::ptr::null_mut(); + crate::workbook::check(unsafe { ffi::xl_rows_open(handle, sheet, &mut rows) })?; Ok(RowCursor { rows, buffer: vec![0; INITIAL_ROW_BUFFER], @@ -243,7 +266,7 @@ impl<'w> RowCursor<'w> { }) } - pub(crate) fn raw(&self) -> *mut crate::XlRows { + pub(crate) fn raw(&self) -> *mut ffi::XlRows { self.rows } @@ -256,26 +279,28 @@ impl<'w> RowCursor<'w> { let mut written: i32 = 0; let capacity = i32::try_from(self.buffer.len()).unwrap_or(i32::MAX); let status = unsafe { - crate::xl_rows_next(self.rows, self.buffer.as_mut_ptr(), capacity, &mut written) + ffi::xl_rows_next(self.rows, self.buffer.as_mut_ptr(), capacity, &mut written) }; match status { - crate::XL_OK => { + ffi::XL_OK => { self.written = if written > 0 { written as usize } else { 0 }; let blob = &self.buffer[..self.written]; - return Some( - RowRef::from_blob(blob).ok_or_else(|| { - Error::from_status(XL_ERROR, "native returned a malformed row blob".to_string()) - }), - ); + return Some(RowRef::from_blob(blob).ok_or_else(|| { + Error::new(ErrorKind::Other, "native returned a malformed row blob") + })); } - crate::XL_EOF => return None, - crate::XL_BUFFER_TOO_SMALL => { - let needed = if written > 0 { written as usize } else { self.buffer.len() * 2 }; + ffi::XL_EOF => return None, + ffi::XL_BUFFER_TOO_SMALL => { + let needed = if written > 0 { + written as usize + } else { + self.buffer.len() * 2 + }; if needed <= self.buffer.len() { - return Some(Err(Error::from_status( - XL_ERROR, - "native asked for a buffer no larger than the current one".to_string(), + return Some(Err(Error::new( + ErrorKind::Other, + "native asked for a buffer no larger than the current one", ))); } self.buffer.resize(needed, 0); @@ -288,74 +313,19 @@ impl<'w> RowCursor<'w> { impl Drop for RowCursor<'_> { fn drop(&mut self) { - unsafe { crate::xl_rows_close(self.rows) }; - } -} - -/// Every row of a sheet, decoded natively in one call. -/// -/// Owns the native allocation and releases it on drop. Rows and cells borrow from it, so they -/// cannot outlive it. -pub struct DecodedRows { - raw: crate::XlRowsDecoded, -} - -impl DecodedRows { - pub(crate) fn new(raw: crate::XlRowsDecoded) -> DecodedRows { - DecodedRows { raw } - } - - #[must_use] - pub fn len(&self) -> usize { - if self.raw.row_count > 0 { self.raw.row_count as usize } else { 0 } - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[must_use] - pub fn get(&self, index: usize) -> Option> { - if index >= self.len() || self.raw.rows.is_null() { - return None; - } - let row = unsafe { &*self.raw.rows.add(index) }; - Some(unsafe { RowRef::from_decoded(row.cells, row.cell_count) }) - } - - pub fn iter(&self) -> impl Iterator> + '_ { - (0..self.len()).filter_map(move |index| self.get(index)) - } -} - -impl Drop for DecodedRows { - fn drop(&mut self) { - unsafe { crate::xl_free_rows(&mut self.raw) }; - } -} - -// SAFETY: plain native allocations, read only through &self, released by xl_free_rows, which any -unsafe impl Send for DecodedRows {} -unsafe impl Sync for DecodedRows {} - -impl std::fmt::Debug for DecodedRows { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DecodedRows").field("len", &self.len()).finish() + unsafe { ffi::xl_rows_close(self.rows) }; } } /// Rows are usually well under this; it only sets how often an oversized sheet costs a retry. const INITIAL_ALL_ROWS_BUFFER: usize = 1024 * 1024; -/// Every row of a sheet, read into one flat buffer by a single `xl_rows_read_all_blob` call. +/// Every row of a sheet, from [`Sheet::read_all`](crate::Sheet::read_all): one native call into +/// one flat buffer this type owns as a plain `Vec`. /// -/// Unlike [`DecodedRows`], the native side allocates nothing per row/cell here - `xl_rows_read_all_blob` -/// writes the same wire format `RowCursor::next_row` decodes, one row after another with a length -/// prefix, into a buffer this type owns as a plain `Vec`. That means no `xl_free_*` call on -/// drop (there is nothing native to release) and one native allocation total instead of one per -/// row - prefer this over [`Sheet::read_all_decoded`](crate::workbook::Sheet::read_all_decoded) -/// unless something specifically needs the decoded-array shape. +/// The native side allocates nothing per row or cell - `xl_rows_read_all_blob` writes the same wire +/// format `RowCursor::next_row` decodes, one row after another with a length prefix - so there is +/// nothing native to release on drop. Rows and cells borrow from it, so they cannot outlive it. pub struct AllRows { buffer: Vec, /// Each row's byte range within `buffer`, already excluding the row's own length prefix - so @@ -364,27 +334,31 @@ pub struct AllRows { } impl AllRows { - pub(crate) fn read(rows: *mut crate::XlRows) -> Result { + pub(crate) fn read(rows: *mut ffi::XlRows) -> Result { let mut buffer = vec![0u8; INITIAL_ALL_ROWS_BUFFER]; loop { let mut written: i32 = 0; let capacity = i32::try_from(buffer.len()).unwrap_or(i32::MAX); let status = unsafe { - crate::xl_rows_read_all_blob(rows, buffer.as_mut_ptr(), capacity, &mut written) + ffi::xl_rows_read_all_blob(rows, buffer.as_mut_ptr(), capacity, &mut written) }; match status { - crate::XL_OK => { + ffi::XL_OK => { buffer.truncate(if written > 0 { written as usize } else { 0 }); let row_ranges = parse_row_ranges(&buffer)?; return Ok(AllRows { buffer, row_ranges }); } - crate::XL_BUFFER_TOO_SMALL => { - let needed = if written > 0 { written as usize } else { buffer.len() * 2 }; + ffi::XL_BUFFER_TOO_SMALL => { + let needed = if written > 0 { + written as usize + } else { + buffer.len() * 2 + }; if needed <= buffer.len() { - return Err(Error::from_status( - XL_ERROR, - "native asked for a buffer no larger than the current one".to_string(), + return Err(Error::new( + ErrorKind::Other, + "native asked for a buffer no larger than the current one", )); } buffer.resize(needed, 0); @@ -438,7 +412,9 @@ fn parse_row_ranges(buffer: &[u8]) -> Result, Error> { return Err(malformed_all_rows_blob()); } let start = offset.checked_add(4).ok_or_else(malformed_all_rows_blob)?; - let end = start.checked_add(row_length as usize).ok_or_else(malformed_all_rows_blob)?; + let end = start + .checked_add(row_length as usize) + .ok_or_else(malformed_all_rows_blob)?; if end > buffer.len() { return Err(malformed_all_rows_blob()); } @@ -453,7 +429,10 @@ fn parse_row_ranges(buffer: &[u8]) -> Result, Error> { } fn malformed_all_rows_blob() -> Error { - Error::from_status(XL_ERROR, "native returned a malformed all-rows blob".to_string()) + Error::new( + ErrorKind::Other, + "native returned a malformed all-rows blob", + ) } #[cfg(test)] @@ -497,7 +476,11 @@ mod tests { #[test] fn iterates_in_order() { - let bytes = blob(&[(0, XL_CELL_STRING, "a"), (1, XL_CELL_STRING, "b"), (2, XL_CELL_STRING, "c")]); + let bytes = blob(&[ + (0, XL_CELL_STRING, "a"), + (1, XL_CELL_STRING, "b"), + (2, XL_CELL_STRING, "c"), + ]); let row = RowRef::from_blob(&bytes).expect("well-formed blob"); let values: Vec<&str> = row.iter().map(|cell| cell.as_str().unwrap()).collect(); assert_eq!(values, ["a", "b", "c"]); @@ -549,8 +532,10 @@ mod tests { let ranges = parse_row_ranges(&bytes).expect("well-formed buffer"); assert_eq!(ranges.len(), 3); - let rows: Vec> = - ranges.iter().map(|&(start, end)| RowRef::from_blob(&bytes[start..end]).unwrap()).collect(); + let rows: Vec> = ranges + .iter() + .map(|&(start, end)| RowRef::from_blob(&bytes[start..end]).unwrap()) + .collect(); assert_eq!(rows[0].len(), 1); assert_eq!(rows[0].get(0).unwrap().as_str().unwrap(), "a"); assert!(rows[1].is_empty()); diff --git a/rust/excelreader/src/source.rs b/rust/excelreader/src/source.rs index cee8f842..cad6b4dd 100644 --- a/rust/excelreader/src/source.rs +++ b/rust/excelreader/src/source.rs @@ -1,13 +1,13 @@ //! Workbooks opened from bytes the caller serves: a positional [`Source`] or any [`Read`]. -use crate::workbook::{check, check_abi_version, Workbook}; -use crate::{Error, OpenOptions, XlSource, XlStream, XlWorkbook, XL_INVALID_ARGUMENT}; +use crate::ffi::{self, XlSource, XlStream}; +use crate::{Error, ErrorKind, OpenOptions, Workbook}; use std::any::Any; use std::ffi::c_void; use std::io::{self, Read}; use std::panic::{catch_unwind, AssertUnwindSafe}; -/// Random-access bytes for [`Workbook::open_source`]. The library calls `read_at` from several +/// Random-access bytes for [`OpenOptions::open_source`]. The library calls `read_at` from several /// threads at once, which is why the trait requires `Send + Sync`. pub trait Source: Send + Sync + 'static { /// Total size in bytes. Called once, while the workbook opens. @@ -18,54 +18,58 @@ pub trait Source: Send + Sync + 'static { fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result; } -impl Workbook { +impl OpenOptions { /// Opens a workbook over `source`, which the library owns from here on and drops once nothing /// opened from the workbook needs it. Every sheet can be read in parallel. - pub fn open_source(source: S, format: i32, options: Option<&OpenOptions>) -> Result { - check_abi_version()?; - let size = source.size().map_err(|error| Error::from_status(XL_INVALID_ARGUMENT, error.to_string()))?; - let length = i64::try_from(size) - .map_err(|_| Error::from_status(XL_INVALID_ARGUMENT, "the source size does not fit in an i64".to_string()))?; - let raw = XlSource { - struct_size: std::mem::size_of::() as i32, - user_data: Box::into_raw(Box::new(source)).cast(), - length, - read_at: Some(read_at::), - release: Some(release::), - }; - let options = options.map(OpenOptions::to_raw); - let options_ptr = options - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - check(unsafe { crate::xl_open_source(&raw, format, options_ptr, &mut handle) })?; - Ok(Workbook::from_handle(handle)) + pub fn open_source(&self, source: S) -> Result { + let size = source + .size() + .map_err(|error| Error::new(ErrorKind::InvalidArgument, error.to_string()))?; + let length = i64::try_from(size).map_err(|_| { + Error::new( + ErrorKind::InvalidArgument, + "the source size does not fit in an i64", + ) + })?; + self.open_raw(|format, options, handle| { + let raw = XlSource { + struct_size: std::mem::size_of::() as i32, + user_data: Box::into_raw(Box::new(source)).cast(), + length, + read_at: Some(read_at::), + release: Some(release::), + }; + unsafe { ffi::xl_open_source(&raw, format, options, handle) } + }) } - /// Opens a workbook over `reader`, which cannot seek. A CSV is read as it arrives, and - /// again from the start until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole first. The library owns `reader` from here on. - pub fn open_reader(reader: R, format: i32, options: Option<&OpenOptions>) -> Result { - check_abi_version()?; - let raw = XlStream { - struct_size: std::mem::size_of::() as i32, - user_data: Box::into_raw(Box::new(reader)).cast(), - read: Some(read::), - release: Some(release::), - }; - let options = options.map(OpenOptions::to_raw); - let options_ptr = options - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - check(unsafe { crate::xl_open_stream(&raw, format, options_ptr, &mut handle) })?; - Ok(Workbook::from_handle(handle)) + /// Opens a workbook over `reader`, which cannot seek. A CSV is read as it arrives, and again + /// from the start until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole first. The + /// library owns `reader` from here on. + pub fn open_reader(&self, reader: R) -> Result { + self.open_raw(|format, options, handle| { + let raw = XlStream { + struct_size: std::mem::size_of::() as i32, + user_data: Box::into_raw(Box::new(reader)).cast(), + read: Some(read::), + release: Some(release::), + }; + unsafe { ffi::xl_open_stream(&raw, format, options, handle) } + }) } } -unsafe extern "C" fn read_at(user_data: *mut c_void, offset: i64, buf: *mut u8, len: i64) -> i64 { +unsafe extern "C" fn read_at( + user_data: *mut c_void, + offset: i64, + buf: *mut u8, + len: i64, +) -> i64 { let source = &*user_data.cast::(); let buf = std::slice::from_raw_parts_mut(buf, len as usize); - report(catch_unwind(AssertUnwindSafe(|| source.read_at(offset as u64, buf)))) + report(catch_unwind(AssertUnwindSafe(|| { + source.read_at(offset as u64, buf) + }))) } unsafe extern "C" fn read(user_data: *mut c_void, buf: *mut u8, len: i64) -> i64 { @@ -80,7 +84,9 @@ unsafe extern "C" fn read(user_data: *mut c_void, buf: *mut u8, len: i6 } unsafe extern "C" fn release(user_data: *mut c_void) { - let _ = catch_unwind(AssertUnwindSafe(|| drop(Box::from_raw(user_data.cast::())))); + let _ = catch_unwind(AssertUnwindSafe(|| { + drop(Box::from_raw(user_data.cast::())) + })); } fn report(outcome: Result, Box>) -> i64 { @@ -89,6 +95,6 @@ fn report(outcome: Result, Box>) -> i64 { Ok(Err(error)) => error.to_string(), Err(_) => "source panicked".to_string(), }; - unsafe { crate::xl_set_source_error(message.as_ptr(), message.len() as i32) }; + unsafe { ffi::xl_set_source_error(message.as_ptr(), message.len() as i32) }; -1 } diff --git a/rust/excelreader/src/temporal.rs b/rust/excelreader/src/temporal.rs index a6cf9ee6..d8857987 100644 --- a/rust/excelreader/src/temporal.rs +++ b/rust/excelreader/src/temporal.rs @@ -24,29 +24,6 @@ pub struct Timestamp { pub micros_since_epoch: i64, } -impl Date { - #[must_use] - pub fn new(days_since_epoch: i32) -> Self { - Self { days_since_epoch } - } -} - -impl Time { - #[must_use] - pub fn new(micros_since_midnight: i64) -> Self { - Self { - micros_since_midnight, - } - } -} - -impl Timestamp { - #[must_use] - pub fn new(micros_since_epoch: i64) -> Self { - Self { micros_since_epoch } - } -} - /// `chrono` interop. Each conversion is total over every value a real workbook can produce, and /// panics only on a value outside `chrono`'s own representable range - which the native converters /// cannot emit, since Excel's serial dates are bounded far inside it. @@ -72,7 +49,9 @@ mod chrono_interop { impl From for Date { fn from(value: NaiveDate) -> Self { - Date::new(value.num_days_from_ce() - DAYS_CE_TO_UNIX_EPOCH) + Date { + days_since_epoch: value.num_days_from_ce() - DAYS_CE_TO_UNIX_EPOCH, + } } } @@ -93,7 +72,9 @@ mod chrono_interop { fn from(value: NaiveTime) -> Self { let secs = i64::from(value.num_seconds_from_midnight()); let micros = i64::from(value.nanosecond() / NANOS_PER_MICRO); - Time::new(secs * MICROS_PER_SEC + micros) + Time { + micros_since_midnight: secs * MICROS_PER_SEC + micros, + } } } @@ -109,7 +90,9 @@ mod chrono_interop { impl From for Timestamp { fn from(value: NaiveDateTime) -> Self { - Timestamp::new(value.and_utc().timestamp_micros()) + Timestamp { + micros_since_epoch: value.and_utc().timestamp_micros(), + } } } } @@ -120,9 +103,27 @@ mod tests { #[test] fn newtypes_carry_the_wire_value_verbatim() { - assert_eq!(Date::new(-1).days_since_epoch, -1); - assert_eq!(Time::new(0).micros_since_midnight, 0); - assert_eq!(Timestamp::new(i64::MIN).micros_since_epoch, i64::MIN); + assert_eq!( + Date { + days_since_epoch: -1 + } + .days_since_epoch, + -1 + ); + assert_eq!( + Time { + micros_since_midnight: 0 + } + .micros_since_midnight, + 0 + ); + assert_eq!( + Timestamp { + micros_since_epoch: i64::MIN + } + .micros_since_epoch, + i64::MIN + ); } #[cfg(feature = "chrono")] @@ -132,35 +133,57 @@ mod tests { #[test] fn date_zero_is_the_unix_epoch() { - let date: NaiveDate = Date::new(0).into(); + let date: NaiveDate = Date { + days_since_epoch: 0, + } + .into(); assert_eq!(date, NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()); } #[test] fn date_round_trips_through_chrono_in_both_directions() { for days in [-25_567, -1, 0, 1, 19_000, 2_958_465 - 25_569] { - let date: NaiveDate = Date::new(days).into(); - assert_eq!(Date::from(date), Date::new(days), "days = {days}"); + let date: NaiveDate = Date { + days_since_epoch: days, + } + .into(); + assert_eq!( + Date::from(date), + Date { + days_since_epoch: days + }, + "days = {days}" + ); } } #[test] fn time_keeps_microsecond_precision() { - let time: NaiveTime = Time::new(13 * 3_600_000_000 + 45 * 60_000_000 + 1).into(); + let time: NaiveTime = Time { + micros_since_midnight: 13 * 3_600_000_000 + 45 * 60_000_000 + 1, + } + .into(); assert_eq!(time, NaiveTime::from_hms_micro_opt(13, 45, 0, 1).unwrap()); assert_eq!( Time::from(time), - Time::new(13 * 3_600_000_000 + 45 * 60_000_000 + 1) + Time { + micros_since_midnight: 13 * 3_600_000_000 + 45 * 60_000_000 + 1 + } ); } #[test] fn timestamp_round_trips_across_the_epoch() { for micros in [-1_000_000, -1, 0, 1, 1_700_000_000_000_000] { - let stamp: NaiveDateTime = Timestamp::new(micros).into(); + let stamp: NaiveDateTime = Timestamp { + micros_since_epoch: micros, + } + .into(); assert_eq!( Timestamp::from(stamp), - Timestamp::new(micros), + Timestamp { + micros_since_epoch: micros + }, "micros = {micros}" ); } diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 14934735..e7d487e0 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -1,14 +1,17 @@ +use crate::ffi::{ + self, XlColumn, XlColumnSpec, XlInferredSchema, XlOpenOptions, XlTable, XlWorkbook, + XL_BUFFER_TOO_SMALL, XL_EOF, XL_OK, +}; use crate::{ - Date, Error, OpenOptions, Time, Timestamp, XlColumn, XlColumnSpec, XlInferredSchema, XlTable, - XlWorkbook, XL_BUFFER_TOO_SMALL, XL_EOF, XL_ERROR, XL_FORMAT_AUTO, XL_OK, XL_T_BOOL, XL_T_DATE, - XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, + AllRows, Date, Error, ErrorKind, OpenOptions, ReadOptions, RowCursor, Time, Timestamp, }; use std::marker::PhantomData; +use std::path::Path; pub(crate) fn last_error(code: i32) -> Error { unsafe { let mut len: i32 = 0; - let ptr = crate::xl_last_error_ptr(&mut len); + let ptr = ffi::xl_last_error_ptr(&mut len); let message = if ptr.is_null() || len <= 0 { "unknown error".to_string() } else { @@ -27,20 +30,37 @@ pub(crate) fn check(code: i32) -> Result<(), Error> { } } +/// The ABI takes every path as UTF-8 bytes. +pub(crate) fn path_str(path: &Path) -> Result<&str, Error> { + path.to_str().ok_or_else(|| { + Error::new( + ErrorKind::InvalidArgument, + format!("path {} is not valid UTF-8", path.display()), + ) + }) +} + +/// The ABI takes every length as `int32`, so an oversized input is rejected rather than truncated. +pub(crate) fn len_i32(len: usize, what: &str) -> Result { + i32::try_from(len).map_err(|_| { + Error::new( + ErrorKind::InvalidArgument, + format!("{what} is {len} bytes, past the ABI's int32 length limit"), + ) + }) +} + /// Copies a native `XlBuffer` into an owned `Vec` and releases the native allocation via -/// `xl_free_buffer` - shared by `writer::write_columns_to_memory`/`write_sheet_to_memory` and -/// `writer_handle::WriterHandle::bytes`, the two places `xl_write_typed_to_memory`/ -/// `xl_write_handle_bytes` hand back an owned buffer. `buffer.data` may be null (an empty result), -/// which `from_raw_parts` cannot take - `slice::from_raw_parts` requires a non-null, well-aligned -/// pointer even for a zero-length slice. -pub(crate) fn buffer_to_vec(mut buffer: crate::XlBuffer) -> Vec { +/// `xl_free_buffer`. `buffer.data` may be null (an empty result), which `from_raw_parts` cannot +/// take even for a zero-length slice. +pub(crate) fn buffer_to_vec(mut buffer: ffi::XlBuffer) -> Vec { let bytes = if buffer.data.is_null() || buffer.len <= 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(buffer.data, buffer.len as usize).to_vec() } }; unsafe { - crate::xl_free_buffer(&mut buffer); + ffi::xl_free_buffer(&mut buffer); } bytes } @@ -53,25 +73,25 @@ pub(crate) fn buffer_to_vec(mut buffer: crate::XlBuffer) -> Vec { /// mismatch would mean reading native memory through the wrong layout, so every constructor calls /// this first. /// -/// The result is cached: it cannot change for the lifetime of the process, and every -/// `Workbook::open` would otherwise pay an FFI call for it. +/// The result is cached: it cannot change for the lifetime of the process, and every open would +/// otherwise pay an FFI call for it. pub(crate) fn check_abi_version() -> Result<(), Error> { use std::sync::OnceLock; static CHECKED: OnceLock> = OnceLock::new(); CHECKED .get_or_init(|| { - let loaded = unsafe { crate::xl_abi_version() }; - if loaded == crate::XL_ABI_VERSION { + let loaded = unsafe { ffi::xl_abi_version() }; + if loaded == ffi::XL_ABI_VERSION { warm_up(); Ok(()) } else { - Err(Error::from_status( - XL_ERROR, + Err(Error::new( + ErrorKind::Other, format!( "ExcelReader native library reports ABI version {loaded}, but this crate \ was built against {}. Update the crate and the native library together.", - crate::XL_ABI_VERSION + ffi::XL_ABI_VERSION ), )) } @@ -97,16 +117,16 @@ fn warm_up() { let mut handle: *mut XlWorkbook = std::ptr::null_mut(); let status = unsafe { - crate::xl_open_memory( + ffi::xl_open_memory( CSV.as_ptr(), CSV.len() as i32, - crate::XL_FORMAT_CSV, + ffi::XL_FORMAT_CSV, std::ptr::null(), &mut handle, ) }; if status == XL_OK && !handle.is_null() { - unsafe { crate::xl_close(handle) }; + unsafe { ffi::xl_close(handle) }; } struct CountRows(i64); @@ -121,11 +141,13 @@ fn warm_up() { } } - let options = crate::CsvParallelOptions { - degree_of_parallelism: 2, - ..crate::CsvParallelOptions::default() - }; - let _ = crate::aggregate::aggregate_csv_memory_unchecked(CSV, || CountRows(0), &options); + let _ = crate::aggregate::aggregate_unchecked( + crate::CsvInput::Bytes(CSV), + &OpenOptions::new(), + 0, + 2, + || CountRows(0), + ); } /// An open workbook. Safe to share between threads: every read goes through a [`Sheet`] and opens @@ -134,113 +156,43 @@ pub struct Workbook { handle: *mut XlWorkbook, } -// SAFETY: ABI v6 documents the workbook handle as usable from several threads at once. The only +// SAFETY: ABI v6 documents the workbook handle as usable from several threads at once. unsafe impl Send for Workbook {} unsafe impl Sync for Workbook {} impl Workbook { - pub(crate) fn from_handle(handle: *mut XlWorkbook) -> Workbook { - Workbook { handle } - } - - /// Opens `path`, sniffing the format and using every library default. - /// - /// Sniffing does NOT detect CSV - open one with [`open_with`](Self::open_with) and - /// [`XL_FORMAT_CSV`](crate::XL_FORMAT_CSV). - pub fn open(path: &str) -> Result { - Self::open_with(path, XL_FORMAT_AUTO, None) - } - - /// Opens `path` with an explicit format and optional [`OpenOptions`]. - pub fn open_with( - path: &str, - format: i32, - options: Option<&OpenOptions>, - ) -> Result { - check_abi_version()?; - let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - let status = unsafe { - crate::xl_open_file( - path.as_ptr(), - path.len() as i32, - format, - raw_ptr, - &mut handle, - ) - }; - check(status)?; - Ok(Workbook { handle }) - } - - /// In-memory equivalent of [`open_with`](Self::open_with): `data` is copied by the native - /// library, so it need not outlive this call. - pub fn open_memory( - data: &[u8], - format: i32, - options: Option<&OpenOptions>, - ) -> Result { - check_abi_version()?; - let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - let status = unsafe { - crate::xl_open_memory( - data.as_ptr(), - data.len() as i32, - format, - raw_ptr, - &mut handle, - ) - }; - check(status)?; - Ok(Workbook { handle }) - } - - /// Number of sheets in the workbook. - pub fn sheet_count(&self) -> Result { - let mut count: i32 = 0; - check(unsafe { crate::xl_sheet_count(self.handle, &mut count) })?; - Ok(count) - } - - /// Name of the sheet at `index`. - pub fn sheet_name_at(&self, index: i32) -> Result { - self.fill_string(|handle, buffer, capacity, out_len| unsafe { - crate::xl_sheet_name_at(handle, index, buffer, capacity, out_len) - }) - } - - /// Every sheet name, in workbook order. - pub fn sheet_names(&self) -> Result, Error> { - (0..self.sheet_count()?) - .map(|index| self.sheet_name_at(index)) - .collect() + /// Opens `path`, sniffing the format and using every library default. Sniffing does not detect + /// CSV: open one with [`OpenOptions::format`] set to [`Format::Csv`](crate::Format::Csv). + pub fn open(path: impl AsRef) -> Result { + OpenOptions::new().open(path) } /// Whether the workbook uses the 1904 date system - needed to interpret raw Excel serial dates. pub fn is_date1904(&self) -> Result { let mut flag: i32 = 0; - check(unsafe { crate::xl_is_date1904(self.handle, &mut flag) })?; + check(unsafe { ffi::xl_is_date1904(self.handle, &mut flag) })?; Ok(flag != 0) } /// The sheet at `index`. An index outside the workbook is an error here, not at the first read. pub fn sheet(&self, index: i32) -> Result, Error> { let mut visibility: i32 = 0; - check(unsafe { crate::xl_sheet_visibility_at(self.handle, index, &mut visibility) })?; - Ok(Sheet { workbook: self, index }) + check(unsafe { ffi::xl_sheet_visibility_at(self.handle, index, &mut visibility) })?; + Ok(Sheet { + workbook: self, + index, + }) } /// Every sheet, in workbook order. Opens nothing. pub fn sheets(&self) -> Result>, Error> { - Ok((0..self.sheet_count()?) - .map(|index| Sheet { workbook: self, index }) + let mut count: i32 = 0; + check(unsafe { ffi::xl_sheet_count(self.handle, &mut count) })?; + Ok((0..count) + .map(|index| Sheet { + workbook: self, + index, + }) .collect()) } @@ -248,9 +200,12 @@ impl Workbook { pub fn sheet_by_name(&self, name: &str) -> Result>, Error> { let mut index: i32 = -1; check(unsafe { - crate::xl_sheet_index(self.handle, name.as_ptr(), name.len() as i32, &mut index) + ffi::xl_sheet_index(self.handle, name.as_ptr(), name.len() as i32, &mut index) })?; - Ok((index >= 0).then_some(Sheet { workbook: self, index })) + Ok((index >= 0).then_some(Sheet { + workbook: self, + index, + })) } /// Shared two-pass buffer dance for the `xl_*` functions that write a UTF-8 name into a caller @@ -272,12 +227,7 @@ impl Workbook { let buffer_slice = &buffer[..len.max(0) as usize]; return str::from_utf8(buffer_slice) .map(|s| s.to_string()) - .map_err(|e| { - Error::from_status( - XL_ERROR, - format!("native library returned a non-UTF-8 name: {e}"), - ) - }); + .map_err(|e| non_utf8_name(&e)); } let mut vec_buffer = vec![0u8; len.max(0) as usize]; status = call( @@ -288,17 +238,53 @@ impl Workbook { ); check(status)?; vec_buffer.truncate(len.max(0) as usize); - String::from_utf8(vec_buffer).map_err(|e| { - Error::from_status( - XL_ERROR, - format!("native library returned a non-UTF-8 name: {e}"), - ) + String::from_utf8(vec_buffer).map_err(|e| non_utf8_name(&e)) + } +} + +fn non_utf8_name(error: &dyn std::fmt::Display) -> Error { + Error::new( + ErrorKind::Other, + format!("native library returned a non-UTF-8 name: {error}"), + ) +} + +impl OpenOptions { + /// Opens the workbook at `path` with these options. + pub fn open(&self, path: impl AsRef) -> Result { + let path = path_str(path.as_ref())?; + let path_len = len_i32(path.len(), "path")?; + self.open_raw(|format, options, handle| unsafe { + ffi::xl_open_file(path.as_ptr(), path_len, format, options, handle) }) } + + /// Opens a workbook held in memory. The native library copies `data`, so it need not outlive + /// this call. + pub fn open_bytes(&self, data: &[u8]) -> Result { + let data_len = len_i32(data.len(), "buffer")?; + self.open_raw(|format, options, handle| unsafe { + ffi::xl_open_memory(data.as_ptr(), data_len, format, options, handle) + }) + } + + /// The tail every open shares: the ABI check, lowering these options, and turning the status + /// and handle into a `Workbook`. `open` runs only after the ABI check passes. + pub(crate) fn open_raw( + &self, + open: impl FnOnce(i32, *const XlOpenOptions, *mut *mut XlWorkbook) -> i32, + ) -> Result { + check_abi_version()?; + let raw = self.to_raw(); + let mut handle: *mut XlWorkbook = std::ptr::null_mut(); + check(open(self.format_raw(), raw.as_ptr(), &mut handle))?; + Ok(Workbook { handle }) + } } /// How a sheet is shown in the application's tab bar. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] pub enum SheetVisibility { Visible, Hidden, @@ -322,22 +308,23 @@ impl<'w> Sheet<'w> { /// The sheet's name. pub fn name(&self) -> Result { - self.workbook.sheet_name_at(self.index) + self.workbook + .fill_string(|handle, buffer, capacity, out_len| unsafe { + ffi::xl_sheet_name_at(handle, self.index, buffer, capacity, out_len) + }) } /// Whether the sheet is visible, hidden or very hidden. An error if the native side reports a /// value this crate does not know. pub fn visibility(&self) -> Result { let mut raw: i32 = 0; - check(unsafe { - crate::xl_sheet_visibility_at(self.workbook.handle, self.index, &mut raw) - })?; + check(unsafe { ffi::xl_sheet_visibility_at(self.workbook.handle, self.index, &mut raw) })?; match raw { - crate::XL_SHEET_VISIBLE => Ok(SheetVisibility::Visible), - crate::XL_SHEET_HIDDEN => Ok(SheetVisibility::Hidden), - crate::XL_SHEET_VERY_HIDDEN => Ok(SheetVisibility::VeryHidden), - other => Err(Error::from_status( - XL_ERROR, + ffi::XL_SHEET_VISIBLE => Ok(SheetVisibility::Visible), + ffi::XL_SHEET_HIDDEN => Ok(SheetVisibility::Hidden), + ffi::XL_SHEET_VERY_HIDDEN => Ok(SheetVisibility::VeryHidden), + other => Err(Error::new( + ErrorKind::Other, format!("native library returned an unknown sheet visibility {other}"), )), } @@ -345,47 +332,58 @@ impl<'w> Sheet<'w> { /// A row-at-a-time reader over this sheet, from its first row. Each call opens an independent /// cursor, so several may be alive at once, on this thread or on others. - pub fn rows(&self) -> Result, Error> { - crate::rows::RowCursor::open(self.workbook.handle, self.index) + pub fn rows(&self) -> Result, Error> { + RowCursor::open(self.workbook.handle, self.index) } - /// Every row of this sheet in one native call, avoiding a round-trip per row. - pub fn read_all_decoded(&self) -> Result { + /// Every row of this sheet, read in one native call into a buffer the result owns outright. + pub fn read_all(&self) -> Result { let cursor = self.rows()?; - let mut raw = crate::XlRowsDecoded { row_count: 0, rows: std::ptr::null_mut() }; - check(unsafe { crate::xl_rows_read_all_decoded(cursor.raw(), &mut raw) })?; - Ok(crate::rows::DecodedRows::new(raw)) + AllRows::read(cursor.raw()) } - /// Every row of this sheet, read in one native call into a buffer this type owns outright. - /// Prefer this over [`read_all_decoded`](Self::read_all_decoded) unless something specifically - /// needs the decoded-array shape. - pub fn read_all_blob(&self) -> Result { - let cursor = self.rows()?; - crate::rows::AllRows::read(cursor.raw()) + /// Schema-driven columnar parse of this sheet into `T`s, matching C++'s `xl::parse_sheet`. + /// The fastest way to read a sheet whose columns you know: it converts on the native side and + /// never formats a cell to text. + /// + /// Uses [`ReadOptions`]'s header row and parallelism; a negative parallelism is an error. + pub fn parse(&self, options: &ReadOptions) -> Result, Error> { + let arena = build_specs::(); + let mut table = empty_table(); + check(unsafe { + ffi::xl_parse_typed( + self.workbook.handle, + self.index, + arena.specs.as_ptr(), + arena.specs.len() as i32, + options.header_row, + options.parallelism, + &mut table, + ) + })?; + TableView::new(table, arena.bindings, "xl_parse_typed") } - /// [`parse_sheet`] delivered a batch at a time. `batch_size` is rows per batch: 0 unbounded, - /// negative an error. The header row is consumed once, here, not re-read per batch. - pub fn typed_chunks( + /// [`parse`](Self::parse) delivered [`ReadOptions`]'s batch size rows at a time; a negative + /// batch size is an error. The header row is consumed once, here, not re-read per batch. + pub fn parse_batches( &self, - header_row: i32, - batch_size: i64, - ) -> Result, Error> { + options: &ReadOptions, + ) -> Result, Error> { let arena = build_specs::(); - let mut reader: *mut crate::XlTypedReader = std::ptr::null_mut(); + let mut reader: *mut ffi::XlTypedReader = std::ptr::null_mut(); check(unsafe { - crate::xl_typed_reader_open( + ffi::xl_typed_reader_open( self.workbook.handle, self.index, arena.specs.as_ptr(), arena.specs.len() as i32, - header_row, - batch_size, + options.header_row, + options.batch_size, &mut reader, ) })?; - Ok(TypedChunks { + Ok(TypedBatches { reader, bindings: arena.bindings, done: false, @@ -393,42 +391,30 @@ impl<'w> Sheet<'w> { }) } - /// Guesses a [`parse_sheet`] schema by sampling this sheet. + /// Guesses a [`parse`](Self::parse) schema by sampling this sheet. /// - /// `header_row` has the same meaning as in [`parse_sheet`] (0 = no header); `sample_size` - /// bounds how many rows after the header are inspected. This is a guess over a sample, not a - /// guarantee - always check it fits before trusting it against the full sheet. + /// `header_row` has the same meaning as in `parse`; `sample_size` bounds how many rows after + /// the header are inspected. With `parse_text`, cells that hold text are typed too - every CSV + /// field, or numbers stored as text: integers, decimals, `true`/`false` and ISO dates or + /// date-times, when the text has exactly that shape (leading-zero codes such as `00123` stay + /// text). This is a guess over a sample, not a guarantee. pub fn infer_schema( &self, header_row: i32, sample_size: i32, + parse_text: bool, ) -> Result, Error> { - self.infer_schema_flags(header_row, sample_size, 0) - } - - /// [`infer_schema`](Self::infer_schema), also typing cells that hold text — every CSV field, - /// or numbers stored as text: integers, decimals, `true`/`false` and ISO dates or date-times, - /// when the text has exactly that shape. Leading-zero codes such as `00123` stay text. - pub fn infer_schema_parse_text( - &self, - header_row: i32, - sample_size: i32, - ) -> Result, Error> { - self.infer_schema_flags(header_row, sample_size, crate::XL_INFER_PARSE_TEXT) - } - - fn infer_schema_flags( - &self, - header_row: i32, - sample_size: i32, - flags: i32, - ) -> Result, Error> { + let flags = if parse_text { + ffi::XL_INFER_PARSE_TEXT + } else { + 0 + }; let mut schema = XlInferredSchema { columns: std::ptr::null_mut(), column_count: 0, }; check(unsafe { - crate::xl_infer_schema( + ffi::xl_infer_schema( self.workbook.handle, self.index, header_row, @@ -439,10 +425,11 @@ impl<'w> Sheet<'w> { })?; let columns = unsafe { copy_inferred(&schema) }; - unsafe { crate::xl_free_schema(&mut schema) }; - Ok(columns) + unsafe { ffi::xl_free_schema(&mut schema) }; + columns } + #[cfg(feature = "arrow")] pub(crate) fn handle(&self) -> *mut XlWorkbook { self.workbook.handle } @@ -455,20 +442,74 @@ impl std::fmt::Debug for Sheet<'_> { } } +/// A column's value type, shared by reading ([`InferredColumn`]) and writing ([`ColumnData`]). +/// +/// [`ColumnData`]: crate::ColumnData +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ColumnType { + String, + I64, + F64, + Bool, + /// Days since 1970-01-01. + Date, + /// Microseconds since midnight. + Time, + /// Microseconds since 1970-01-01T00:00:00Z. + Timestamp, +} + +impl ColumnType { + pub(crate) fn raw(self) -> i32 { + match self { + ColumnType::String => ffi::XL_T_STRING, + ColumnType::I64 => ffi::XL_T_I64, + ColumnType::F64 => ffi::XL_T_F64, + ColumnType::Bool => ffi::XL_T_BOOL, + ColumnType::Date => ffi::XL_T_DATE, + ColumnType::Time => ffi::XL_T_TIME, + ColumnType::Timestamp => ffi::XL_T_TIMESTAMP, + } + } + + fn from_raw(value: i32) -> Option { + match value { + ffi::XL_T_STRING => Some(ColumnType::String), + ffi::XL_T_I64 => Some(ColumnType::I64), + ffi::XL_T_F64 => Some(ColumnType::F64), + ffi::XL_T_BOOL => Some(ColumnType::Bool), + ffi::XL_T_DATE => Some(ColumnType::Date), + ffi::XL_T_TIME => Some(ColumnType::Time), + ffi::XL_T_TIMESTAMP => Some(ColumnType::Timestamp), + _ => None, + } + } +} + /// Deep-copies a native-owned inferred schema into owned Rust values, so the caller can free the /// native allocation immediately. /// /// # Safety /// `schema` must be one `xl_infer_schema` returned `XL_OK` for, not yet passed to `xl_free_schema`. -unsafe fn copy_inferred(schema: &XlInferredSchema) -> Vec { +unsafe fn copy_inferred(schema: &XlInferredSchema) -> Result, Error> { if schema.columns.is_null() || schema.column_count <= 0 { - return Vec::new(); + return Ok(Vec::new()); } let specs = std::slice::from_raw_parts(schema.columns, schema.column_count as usize); specs .iter() - .map(|spec| InferredColumn { - name: if spec.name_count <= 0 || spec.names.is_null() || spec.name_lens.is_null() { + .map(|spec| { + let column_type = ColumnType::from_raw(spec.r#type).ok_or_else(|| { + Error::new( + ErrorKind::Other, + format!( + "native library inferred an unknown column type {}", + spec.r#type + ), + ) + })?; + let name = if spec.name_count <= 0 || spec.names.is_null() || spec.name_lens.is_null() { None } else { let name_ptr = *spec.names; @@ -479,22 +520,25 @@ unsafe fn copy_inferred(schema: &XlInferredSchema) -> Vec { let bytes = std::slice::from_raw_parts(name_ptr, name_len as usize); Some(String::from_utf8_lossy(bytes).into_owned()) } - }, - index: spec.index, - column_type: spec.r#type, - nullable: spec.nullable != 0, + }; + Ok(InferredColumn { + name, + index: spec.index, + column_type, + nullable: spec.nullable != 0, + }) }) .collect() } /// One column guessed by [`Sheet::infer_schema`]. #[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] pub struct InferredColumn { /// Header text, or `None` when the column must be resolved by [`index`](Self::index) instead. pub name: Option, pub index: i32, - /// One of the `XL_T_*` constants. - pub column_type: i32, + pub column_type: ColumnType, pub nullable: bool, } @@ -502,7 +546,7 @@ impl Drop for Workbook { fn drop(&mut self) { if !self.handle.is_null() { unsafe { - crate::xl_close(self.handle); + ffi::xl_close(self.handle); } self.handle = std::ptr::null_mut(); } @@ -519,11 +563,11 @@ impl std::fmt::Debug for Workbook { } } -/// One field <-> column binding for `T`. Construct via `ExcelMapper::bindings()`. +/// One field <-> column binding for `T`, produced by `#[derive(ExcelMapper)]`. pub struct ColumnBinding { /// Candidate header names, in priority order — the first one present in the header row wins. pub names: &'static [&'static str], - pub xl_type: i32, + pub column_type: ColumnType, pub assign: fn(&mut T, &XlColumn, i64), } @@ -536,10 +580,10 @@ impl Clone for ColumnBinding { impl Copy for ColumnBinding {} -/// Implemented by any struct `parse_sheet` can populate from a `parse_typed` result. Implement -/// `bindings()` by hand (mirroring `xl::ExcelMapper` on the C++ side), or derive it with -/// `#[derive(ExcelMapper)]` (see `excelreader_derive`). +/// Implemented by any struct [`Sheet::parse`] can populate. Derive it with +/// `#[derive(ExcelMapper)]`, which also implements [`ExcelWriter`](crate::ExcelWriter). pub trait ExcelMapper: Sized { + #[doc(hidden)] fn bindings() -> Vec>; } @@ -559,8 +603,8 @@ fn is_valid(col: &XlColumn, row: i64) -> bool { /// /// These are safe `pub` functions that index into raw native buffers, so this check is what makes /// them sound: without it a `row` past the column returns whatever sits after the allocation. The -/// type half catches the other class of mistake - reading an `XL_T_F64` buffer as `i64` is an -/// `ExcelMapper::bindings()` bug, not recoverable input. +/// type half catches the other class of mistake - reading an `XL_T_F64` buffer as `i64` is a +/// bindings bug, not recoverable input. #[inline] fn check_access(col: &XlColumn, row: i64, expected_type: i32, accessor: &str) { assert_eq!( @@ -580,13 +624,12 @@ fn check_access(col: &XlColumn, row: i64, expected_type: i32, accessor: &str) { /// The ABI documents every string it returns as UTF-8, but this validates rather than trusting it: /// the bytes cross an FFI boundary from a binary resolved at build time, and `from_utf8_unchecked` /// on a value that turns out not to be UTF-8 is undefined behavior, not merely a wrong answer. -/// Validation is a linear scan over bytes already in cache. /// /// # Panics -/// If `col` is not an `XL_T_STRING` column, if `row` is out of bounds, or if the native library -/// returned bytes that are not valid UTF-8 (an ABI contract violation). +/// If `col` is not a string column, if `row` is out of bounds, or if the native library returned +/// bytes that are not valid UTF-8 (an ABI contract violation). pub fn column_str(col: &XlColumn, row: i64) -> &str { - check_access(col, row, XL_T_STRING, "column_str"); + check_access(col, row, ffi::XL_T_STRING, "column_str"); unsafe { let offsets = col.values as *const i32; let start = *offsets.offset(row as isize); @@ -599,50 +642,64 @@ pub fn column_str(col: &XlColumn, row: i64) -> &str { } /// # Panics -/// If `col` is not an `XL_T_I64` column, or `row` is out of bounds. +/// If `col` is not an `i64` column, or `row` is out of bounds. pub fn column_i64(col: &XlColumn, row: i64) -> i64 { - check_access(col, row, XL_T_I64, "column_i64"); + check_access(col, row, ffi::XL_T_I64, "column_i64"); unsafe { *(col.values as *const i64).offset(row as isize) } } /// # Panics -/// If `col` is not an `XL_T_F64` column, or `row` is out of bounds. +/// If `col` is not an `f64` column, or `row` is out of bounds. pub fn column_f64(col: &XlColumn, row: i64) -> f64 { - check_access(col, row, XL_T_F64, "column_f64"); + check_access(col, row, ffi::XL_T_F64, "column_f64"); unsafe { *(col.values as *const f64).offset(row as isize) } } /// # Panics -/// If `col` is not an `XL_T_BOOL` column, or `row` is out of bounds. +/// If `col` is not a bool column, or `row` is out of bounds. pub fn column_bool(col: &XlColumn, row: i64) -> bool { - check_access(col, row, XL_T_BOOL, "column_bool"); + check_access(col, row, ffi::XL_T_BOOL, "column_bool"); unsafe { *(col.values as *const u8).offset(row as isize) != 0 } } /// # Panics -/// If `col` is not an `XL_T_DATE` column, or `row` is out of bounds. +/// If `col` is not a date column, or `row` is out of bounds. pub fn column_date(col: &XlColumn, row: i64) -> Date { - check_access(col, row, XL_T_DATE, "column_date"); - Date::new(unsafe { *(col.values as *const i32).offset(row as isize) }) + check_access(col, row, ffi::XL_T_DATE, "column_date"); + Date { + days_since_epoch: unsafe { *(col.values as *const i32).offset(row as isize) }, + } } /// # Panics -/// If `col` is not an `XL_T_TIME` column, or `row` is out of bounds. +/// If `col` is not a time column, or `row` is out of bounds. pub fn column_time(col: &XlColumn, row: i64) -> Time { - check_access(col, row, XL_T_TIME, "column_time"); - Time::new(unsafe { *(col.values as *const i64).offset(row as isize) }) + check_access(col, row, ffi::XL_T_TIME, "column_time"); + Time { + micros_since_midnight: unsafe { *(col.values as *const i64).offset(row as isize) }, + } } /// # Panics -/// If `col` is not an `XL_T_TIMESTAMP` column, or `row` is out of bounds. +/// If `col` is not a timestamp column, or `row` is out of bounds. pub fn column_timestamp(col: &XlColumn, row: i64) -> Timestamp { - check_access(col, row, XL_T_TIMESTAMP, "column_timestamp"); - Timestamp::new(unsafe { *(col.values as *const i64).offset(row as isize) }) + check_access(col, row, ffi::XL_T_TIMESTAMP, "column_timestamp"); + Timestamp { + micros_since_epoch: unsafe { *(col.values as *const i64).offset(row as isize) }, + } } -/// Owns a `parse_typed` result. Frees the native table on `Drop`. `iter()` builds one `T` per call -/// to `next()` from the columnar buffers - no upfront `Vec` allocation beyond what -/// `xl_parse_typed` itself already made. +fn empty_table() -> XlTable { + XlTable { + column_count: 0, + row_count: 0, + columns: std::ptr::null_mut(), + } +} + +/// Owns a typed parse result. Frees the native table on `Drop`. `iter()` builds one `T` per call +/// to `next()` from the columnar buffers - no upfront `Vec` allocation beyond what the native +/// parse itself already made. pub struct TableView { table: XlTable, bindings: Vec>, @@ -650,6 +707,31 @@ pub struct TableView { } impl TableView { + /// Takes ownership of `table`, freeing it and failing if its column count disagrees with the + /// bindings it will be read through. + fn new( + mut table: XlTable, + bindings: Vec>, + source: &str, + ) -> Result { + if table.column_count as usize != bindings.len() { + let column_count = table.column_count; + unsafe { ffi::xl_free_table(&mut table) }; + return Err(Error::new( + ErrorKind::Other, + format!( + "{source} returned {column_count} columns for {} specs", + bindings.len() + ), + )); + } + Ok(TableView { + table, + bindings, + _marker: PhantomData, + }) + } + pub fn len(&self) -> i64 { self.table.row_count } @@ -692,12 +774,13 @@ impl TableView { impl Drop for TableView { fn drop(&mut self) { unsafe { - crate::xl_free_table(&mut self.table); + ffi::xl_free_table(&mut self.table); } } } // SAFETY: plain native allocations, read only through &self, released by xl_free_table, which any +// thread may call. unsafe impl Send for TableView {} unsafe impl Sync for TableView {} @@ -713,17 +796,18 @@ impl std::fmt::Debug for TableView { } /// Iterator over the batches of one chunked typed read. Closes the native reader on `Drop`. -pub struct TypedChunks<'a, T: ExcelMapper> { - reader: *mut crate::XlTypedReader, +pub struct TypedBatches<'a, T: ExcelMapper> { + reader: *mut ffi::XlTypedReader, bindings: Vec>, done: bool, _workbook: PhantomData<&'a Workbook>, } // SAFETY: a typed reader owns its native enumerator; ABI v6 lets it be used from any one thread -unsafe impl Send for TypedChunks<'_, T> {} +// at a time. +unsafe impl Send for TypedBatches<'_, T> {} -impl Iterator for TypedChunks<'_, T> { +impl Iterator for TypedBatches<'_, T> { type Item = Result, Error>; fn next(&mut self) -> Option { @@ -731,12 +815,8 @@ impl Iterator for TypedChunks<'_, T> { return None; } - let mut table = XlTable { - column_count: 0, - row_count: 0, - columns: std::ptr::null_mut(), - }; - let status = unsafe { crate::xl_typed_reader_next(self.reader, &mut table) }; + let mut table = empty_table(); + let status = unsafe { ffi::xl_typed_reader_next(self.reader, &mut table) }; if status == XL_EOF { self.done = true; return None; @@ -746,38 +826,23 @@ impl Iterator for TypedChunks<'_, T> { return Some(Err(last_error(status))); } - if table.column_count as usize != self.bindings.len() { - let column_count = table.column_count; - unsafe { crate::xl_free_table(&mut table) }; - self.done = true; - return Some(Err(Error::from_status( - XL_ERROR, - format!( - "xl_typed_reader_next returned {column_count} columns for {} specs", - self.bindings.len() - ), - ))); - } - - // ponytail: one small Vec of function pointers cloned per batch, not per row. Give - Some(Ok(TableView { - table, - bindings: self.bindings.clone(), - _marker: PhantomData, - })) + // ponytail: one small Vec of function pointers cloned per batch, not per row. + let view = TableView::new(table, self.bindings.clone(), "xl_typed_reader_next"); + self.done = view.is_err(); + Some(view) } } -impl Drop for TypedChunks<'_, T> { +impl Drop for TypedBatches<'_, T> { fn drop(&mut self) { - unsafe { crate::xl_typed_reader_close(self.reader) }; + unsafe { ffi::xl_typed_reader_close(self.reader) }; } } /// Progress only - the reader handle is opaque. Needed for `expect_err` on a `Result`. -impl std::fmt::Debug for TypedChunks<'_, T> { +impl std::fmt::Debug for TypedBatches<'_, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TypedChunks") + f.debug_struct("TypedBatches") .field("columns", &self.bindings.len()) .field("done", &self.done) .finish_non_exhaustive() @@ -840,7 +905,7 @@ pub(crate) fn build_specs() -> SpecArena { name_lens: name_lens[i].as_ptr(), name_count: b.names.len() as i32, index: 0, - r#type: b.xl_type, + r#type: b.column_type.raw(), nullable: 1, }) .collect(); @@ -851,60 +916,3 @@ pub(crate) fn build_specs() -> SpecArena { _name_lens: name_lens, } } - -/// Schema-driven columnar parse of one sheet, matching C++'s `xl::parse_sheet`. -pub fn parse_sheet( - sheet: Sheet<'_>, - header_row: i32, -) -> Result, Error> { - parse_sheet_parallel(sheet, header_row, 1) -} - -/// [`parse_sheet`] with a degree of parallelism: `0` uses every processor, `1` is sequential, `n` -/// up to `n` threads, negative is an error. Only CSV is split; other formats read sequentially. -/// The table equals `parse_sheet`'s. -pub fn parse_sheet_parallel( - sheet: Sheet<'_>, - header_row: i32, - degree_of_parallelism: i32, -) -> Result, Error> { - let arena = build_specs::(); - let bindings = arena.bindings; - let mut table = XlTable { - column_count: 0, - row_count: 0, - columns: std::ptr::null_mut(), - }; - unsafe { - let status = crate::xl_parse_typed( - sheet.handle(), - sheet.index(), - arena.specs.as_ptr(), - arena.specs.len() as i32, - header_row, - degree_of_parallelism, - &mut table, - ); - if status != XL_OK { - return Err(last_error(status)); - } - } - - if table.column_count as usize != bindings.len() { - let column_count = table.column_count; - unsafe { crate::xl_free_table(&mut table) }; - return Err(Error::from_status( - XL_ERROR, - format!( - "xl_parse_typed returned {column_count} columns for {} specs", - bindings.len() - ), - )); - } - - Ok(TableView { - table, - bindings, - _marker: PhantomData, - }) -} diff --git a/rust/excelreader/src/writer.rs b/rust/excelreader/src/writer.rs index 69a95618..09f05976 100644 --- a/rust/excelreader/src/writer.rs +++ b/rust/excelreader/src/writer.rs @@ -1,4 +1,4 @@ -//! Writing a columnar table to a workbook file, through the ABI's single `xl_write_typed` export. +//! Writing a columnar table to a workbook, through the ABI's single-shot `xl_write_typed` exports. //! //! Two layers sit here. [`write_columns`] takes buffers the caller already owns and hands them //! straight to the ABI - the borrow is the whole point, and the lifetimes on [`Column`] are what @@ -6,16 +6,16 @@ //! something the compiler enforces. [`write_sheet`] sits on top for callers holding a slice of //! structs, and pays exactly one transpose to get them into columns. -use crate::workbook::{check, check_abi_version}; -use crate::{ - Error, WriteOptions, XlColumn, XlColumnSpec, XlTable, XL_FORMAT_AUTO, XL_FORMAT_CSV, - XL_FORMAT_XLS, XL_FORMAT_XLSB, XL_FORMAT_XLSX, XL_INVALID_ARGUMENT, XL_T_BOOL, XL_T_DATE, - XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, -}; +use crate::ffi::{self, XlColumn, XlColumnSpec, XlTable}; +use crate::workbook::{buffer_to_vec, check, check_abi_version, len_i32, path_str}; +use crate::{ColumnType, Error, ErrorKind, WriteOptions}; +use std::io::Write; use std::os::raw::c_void; +use std::path::Path; /// One column's buffers, borrowed from the caller. Each variant's slice is that column type's /// exact wire layout (see `excelreader.h`), so nothing is converted on the way out. +#[non_exhaustive] pub enum ColumnData<'a> { /// `offsets` has `rows + 1` entries into `data`, which is every row's UTF-8 bytes /// concatenated. Unlike a table returned by the reader, `data` need not be interior to @@ -56,17 +56,17 @@ impl ColumnData<'_> { } /// The `XL_T_*` tag this variant writes as. - #[must_use] - pub fn xl_type(&self) -> i32 { + pub(crate) fn xl_type(&self) -> i32 { match self { - ColumnData::Str { .. } => XL_T_STRING, - ColumnData::I64(_) => XL_T_I64, - ColumnData::F64(_) => XL_T_F64, - ColumnData::Bool(_) => XL_T_BOOL, - ColumnData::Date(_) => XL_T_DATE, - ColumnData::Time(_) => XL_T_TIME, - ColumnData::Timestamp(_) => XL_T_TIMESTAMP, + ColumnData::Str { .. } => ColumnType::String, + ColumnData::I64(_) => ColumnType::I64, + ColumnData::F64(_) => ColumnType::F64, + ColumnData::Bool(_) => ColumnType::Bool, + ColumnData::Date(_) => ColumnType::Date, + ColumnData::Time(_) => ColumnType::Time, + ColumnData::Timestamp(_) => ColumnType::Timestamp, } + .raw() } fn pointers(&self) -> (*const c_void, *const u8, i64) { @@ -87,6 +87,7 @@ impl ColumnData<'_> { } /// One input column: a name, its buffers, and an optional validity bitmap. +#[non_exhaustive] pub struct Column<'a> { /// The header text. `None` in EVERY column means no header row is written; mixing `Some` and /// `None` across the set is an error, not a partial header. @@ -96,7 +97,24 @@ pub struct Column<'a> { pub validity: Option<&'a [u8]>, } -impl Column<'_> { +impl<'a> Column<'a> { + /// A column with no nulls. `name` is the header text; see [`name`](Self::name). + #[must_use] + pub fn new(name: Option<&'a str>, data: ColumnData<'a>) -> Self { + Column { + name, + data, + validity: None, + } + } + + /// Sets the LSB-first validity bitmap: bit `r` set = row `r` is valid. + #[must_use] + pub fn validity(mut self, bitmap: &'a [u8]) -> Self { + self.validity = Some(bitmap); + self + } + fn to_raw(&self) -> XlColumn { let (values, data, data_len) = self.data.pointers(); XlColumn { @@ -110,27 +128,8 @@ impl Column<'_> { } } -/// Infers an `XL_FORMAT_*` from a path's extension, case-insensitively. Returns -/// [`XL_FORMAT_AUTO`] when the extension is absent or unrecognized - and since `xl_write_typed` -/// rejects `AUTO` with a message of its own, an unrecognized path fails the write rather than -/// silently picking a format. -#[must_use] -pub fn format_from_path(path: &str) -> i32 { - let name = path.rsplit(['/', '\\']).next().unwrap_or(path); - let Some((_, extension)) = name.rsplit_once('.') else { - return XL_FORMAT_AUTO; - }; - match extension.to_ascii_lowercase().as_str() { - "xlsx" => XL_FORMAT_XLSX, - "xlsb" => XL_FORMAT_XLSB, - "xls" => XL_FORMAT_XLS, - "csv" => XL_FORMAT_CSV, - _ => XL_FORMAT_AUTO, - } -} - fn invalid(message: String) -> Error { - Error::from_status(XL_INVALID_ARGUMENT, message) + Error::new(ErrorKind::InvalidArgument, message) } /// Returns the row count every column agreed on, or the first problem found. @@ -191,23 +190,18 @@ fn validate(columns: &[Column<'_>]) -> Result { Ok(rows) } -/// Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle -/// exists before or after, and every buffer reachable from `columns` and `options` is borrowed for -/// the duration of the call and never freed by the native library. -/// -/// `format` must be one of `XL_FORMAT_XLS`/`XLSX`/`XLSB`/`CSV`. [`XL_FORMAT_AUTO`] is an error: a -/// file being created has no signature bytes to sniff. Use [`format_from_path`] to infer one. -/// -/// On failure the destination may exist and be incomplete; cleaning it up is the caller's. -pub fn write_columns( - path: &str, - format: i32, - columns: &[Column<'_>], - options: Option<&WriteOptions>, -) -> Result<(), Error> { - check_abi_version()?; - let row_count = validate(columns)?; +/// The ABI's column specs and table for `columns`, borrowing every buffer. `specs` and `table` +/// point into the other three fields, so the whole value must outlive the FFI call. +struct RawTable { + specs: Vec, + table: XlTable, + _columns: Vec, + _names: Vec<*const u8>, + _name_lens: Vec, +} +fn raw_table(columns: &[Column<'_>]) -> Result { + let row_count = validate(columns)?; let names: Vec<*const u8> = columns .iter() .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) @@ -229,88 +223,89 @@ pub fn write_columns( }) .collect(); let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); - let table = XlTable { column_count: columns.len() as i32, row_count, columns: raw_columns.as_ptr().cast_mut(), }; - let raw_options = options.map(WriteOptions::to_raw); - let options_ptr = crate::options::ptr_or_null(&raw_options); + Ok(RawTable { + specs, + table, + _columns: raw_columns, + _names: names, + _name_lens: name_lens, + }) +} - let status = unsafe { - crate::xl_write_typed( +/// Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle +/// exists before or after, and every buffer reachable from `columns` and `options` is borrowed for +/// the duration of the call and never freed by the native library. +/// +/// The format is [`WriteOptions::format`], or inferred from the extension when that is +/// [`Format::Auto`](crate::Format::Auto) - an unrecognized extension is then an error, since a file +/// being created has no signature bytes to sniff. +/// +/// On failure the destination may exist and be incomplete; cleaning it up is the caller's. +pub fn write_columns( + path: impl AsRef, + columns: &[Column<'_>], + options: &WriteOptions, +) -> Result<(), Error> { + check_abi_version()?; + let path = path.as_ref(); + let format = options.format_for(Some(path))?; + let path = path_str(path)?; + let path_len = len_i32(path.len(), "path")?; + let raw = raw_table(columns)?; + let raw_options = options.to_raw(); + check(unsafe { + ffi::xl_write_typed( path.as_ptr(), - path.len() as i32, + path_len, format, - specs.as_ptr(), - &table, - options_ptr, + raw.specs.as_ptr(), + &raw.table, + &raw_options, ) - }; - check(status) + }) } -/// In-memory equivalent of [`write_columns`]: same validation and column lowering, but the -/// workbook is built in memory and returned as bytes instead of being written to a path - so, -/// unlike [`write_columns`], there is no path to infer a format from and `format` is always -/// required. -/// -/// # Errors -/// Anything [`write_columns`] reports. -pub fn write_columns_to_memory( - format: i32, +/// [`write_columns`] to any [`Write`]: the workbook is built in memory, then written to `writer` +/// in one `write_all`. [`WriteOptions::format`] must be set - there is no extension to infer one +/// from. +pub fn write_columns_to( + mut writer: impl Write, columns: &[Column<'_>], - options: Option<&WriteOptions>, -) -> Result, Error> { + options: &WriteOptions, +) -> Result<(), Error> { check_abi_version()?; - let row_count = validate(columns)?; - - let names: Vec<*const u8> = columns - .iter() - .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) - .collect(); - let name_lens: Vec = columns - .iter() - .map(|c| c.name.map_or(0, |n| n.len() as i32)) - .collect(); - let specs: Vec = columns - .iter() - .enumerate() - .map(|(index, column)| XlColumnSpec { - names: &names[index], - name_lens: &name_lens[index], - name_count: i32::from(column.name.is_some()), - index: 0, - r#type: column.data.xl_type(), - nullable: 0, - }) - .collect(); - let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); - - let table = XlTable { - column_count: columns.len() as i32, - row_count, - columns: raw_columns.as_ptr().cast_mut(), - }; - let raw_options = options.map(WriteOptions::to_raw); - let options_ptr = crate::options::ptr_or_null(&raw_options); - - let mut buffer = crate::XlBuffer { + let format = options.format_for(None)?; + let raw = raw_table(columns)?; + let raw_options = options.to_raw(); + let mut buffer = ffi::XlBuffer { data: std::ptr::null_mut(), len: 0, }; - let status = unsafe { - crate::xl_write_typed_to_memory(format, specs.as_ptr(), &table, options_ptr, &mut buffer) - }; - check(status)?; - Ok(crate::workbook::buffer_to_vec(buffer)) + check(unsafe { + ffi::xl_write_typed_to_memory( + format, + raw.specs.as_ptr(), + &raw.table, + &raw_options, + &mut buffer, + ) + })?; + let bytes = buffer_to_vec(buffer); + writer + .write_all(&bytes) + .and_then(|()| writer.flush()) + .map_err(|e| Error::io(&e)) } /// Wraps a finished plaintext XLSX/XLSB package at `package_path` in an agile-encrypted (ECMA-376 /// 4.4) CFB container, written to `destination_path` (overwriting an existing file). The result -/// opens with the same password via [`crate::OpenOptions::password`]. Encryption parameters are -/// fixed at Excel's own defaults - there are no options. +/// opens with the same password via [`OpenOptions::password`](crate::OpenOptions::password). +/// Encryption parameters are fixed at Excel's own defaults - there are no options. /// /// `package_path` is read twice, so it must already be a finished file (write it with /// [`write_columns`]/[`write_sheet`] first). @@ -318,23 +313,31 @@ pub fn write_columns_to_memory( /// # Errors /// `package_path` does not exist, is not readable, or is not a valid plaintext OOXML package; /// `password` is empty; or the usual file I/O failures on either path. -pub fn encrypt_package(package_path: &str, destination_path: &str, password: &str) -> Result<(), Error> { +pub fn encrypt_package( + package_path: impl AsRef, + destination_path: impl AsRef, + password: &str, +) -> Result<(), Error> { check_abi_version()?; - let status = unsafe { - crate::xl_encrypt_package( + let package_path = path_str(package_path.as_ref())?; + let destination_path = path_str(destination_path.as_ref())?; + let package_path_len = len_i32(package_path.len(), "package_path")?; + let destination_path_len = len_i32(destination_path.len(), "destination_path")?; + let password_len = len_i32(password.len(), "password")?; + check(unsafe { + ffi::xl_encrypt_package( package_path.as_ptr(), - package_path.len() as i32, + package_path_len, destination_path.as_ptr(), - destination_path.len() as i32, + destination_path_len, password.as_ptr(), - password.len() as i32, + password_len, ) - }; - check(status) + }) } -/// The owning twin of [`ColumnData`], produced by [`ExcelWriter::to_columns`]. A transposed range -/// of structs has to own its columns somewhere; this is that somewhere. +/// The owning twin of [`ColumnData`], produced by `#[derive(ExcelMapper)]`'s [`ExcelWriter`] impl. +/// A transposed range of structs has to own its columns somewhere; this is that somewhere. pub enum OwnedColumnData { Str { offsets: Vec, data: Vec }, I64(Vec), @@ -356,9 +359,7 @@ pub struct OwnedColumn { } impl OwnedColumn { - /// Borrows this column in the shape [`write_columns`] takes. - #[must_use] - pub fn as_column(&self) -> Column<'_> { + fn as_column(&self) -> Column<'_> { let data = match &self.data { OwnedColumnData::Str { offsets, data } => ColumnData::Str { offsets, data }, OwnedColumnData::I64(v) => ColumnData::I64(v), @@ -411,78 +412,61 @@ pub fn set_valid(validity: &mut [u8], row: usize) { validity[row / 8] |= 1 << (row % 8); } -/// Implemented by any struct [`write_sheet`] can write. Derive it with -/// `#[derive(ExcelMapper)]`, which emits this alongside the reading half, or write it by hand. +/// Implemented by any struct [`write_sheet`] can write. Derive it with `#[derive(ExcelMapper)]`, +/// which emits this alongside the reading half. pub trait ExcelWriter: Sized { /// Transposes `rows` into one column per field, in field order. /// /// # Errors /// Only for values the ABI cannot represent: a string column past 2 GiB, or an integer that /// does not fit `i64`. Ordinary data never fails here. + #[doc(hidden)] fn to_columns(rows: &[Self]) -> Result, Error>; } /// Writes `rows` to `path` as a single sheet, using the same field mapping -/// [`parse_sheet`](crate::workbook::parse_sheet) reads with. +/// [`Sheet::parse`](crate::Sheet::parse) reads with. The format is chosen as in +/// [`write_columns`]. /// /// `rows` is walked ONCE and each field appended to its own column buffer. That transpose is the /// only copy this makes - it is what the ABI's columnar shape costs a row-shaped caller. If you /// already hold columnar buffers, call [`write_columns`] and pay nothing. -/// -/// # Errors -/// Anything [`ExcelWriter::to_columns`] or [`write_columns`] reports. pub fn write_sheet( - path: &str, - format: i32, + path: impl AsRef, rows: &[T], - options: Option<&WriteOptions>, + options: &WriteOptions, ) -> Result<(), Error> { let owned = T::to_columns(rows)?; let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); - write_columns(path, format, &borrowed, options) + write_columns(path, &borrowed, options) } -/// In-memory equivalent of [`write_sheet`]: same transpose, but returns bytes instead of writing -/// to a path - see [`write_columns_to_memory`] for why `format` is always required here. -/// -/// # Errors -/// Anything [`ExcelWriter::to_columns`] or [`write_columns_to_memory`] reports. -pub fn write_sheet_to_memory( - format: i32, +/// [`write_sheet`] to any [`Write`], as [`write_columns_to`] does it. +pub fn write_sheet_to( + writer: impl Write, rows: &[T], - options: Option<&WriteOptions>, -) -> Result, Error> { + options: &WriteOptions, +) -> Result<(), Error> { let owned = T::to_columns(rows)?; let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); - write_columns_to_memory(format, &borrowed, options) + write_columns_to(writer, &borrowed, options) } #[cfg(test)] mod tests { use super::*; - #[test] - fn format_is_inferred_from_the_extension() { - assert_eq!(format_from_path("out.xlsx"), XL_FORMAT_XLSX); - assert_eq!(format_from_path("out.XLSB"), XL_FORMAT_XLSB); - assert_eq!(format_from_path("out.xls"), XL_FORMAT_XLS); - assert_eq!(format_from_path("out.csv"), XL_FORMAT_CSV); - assert_eq!(format_from_path("out.txt"), XL_FORMAT_AUTO); - assert_eq!(format_from_path("out"), XL_FORMAT_AUTO); - assert_eq!(format_from_path("v1.2/report"), XL_FORMAT_AUTO); - } - #[test] fn validate_rejects_a_short_validity_bitmap() { let values = [1i64, 2, 3, 4, 5, 6, 7, 8, 9]; - let bitmap = [0u8]; + let bitmap = [0u8]; let columns = [Column { name: Some("a"), data: ColumnData::I64(&values), validity: Some(&bitmap), }]; let error = validate(&columns).expect_err("a 1-byte bitmap cannot cover 9 rows"); - assert_eq!(error.code(), XL_INVALID_ARGUMENT); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); } #[test] @@ -531,6 +515,7 @@ mod tests { "3 rows next to a 2-row column must be rejected" ); } + #[test] fn push_str_appends_bytes_and_one_offset_per_value() { let mut offsets = vec![0i32]; diff --git a/rust/excelreader/src/writer_handle.rs b/rust/excelreader/src/writer_handle.rs index 1f8abdf7..da518240 100644 --- a/rust/excelreader/src/writer_handle.rs +++ b/rust/excelreader/src/writer_handle.rs @@ -1,90 +1,79 @@ //! Row-by-row streaming write, through the ABI's `xl_writer_handle`. //! -//! [`WriterHandle`] is the streaming counterpart to [`crate::writer::write_columns`]/ -//! [`crate::writer::write_sheet`]: instead of materializing a whole table up front, it writes -//! directly as each call arrives - one sheet and one row open at a time. Call order mirrors the C -//! ABI (see `xl_writer_handle` in `excelreader.h`): `open`/`open_with`/`open_memory`, then per -//! sheet `start_sheet`..`end_sheet`, each containing `start_row`..`end_row` with one `write_*` -//! call per cell in between, left to right. A call out of order returns an `Err` rather than -//! corrupting output, and the handle stays usable afterward - fix the call order and continue, or -//! let `Drop` discard it. - -use crate::workbook::{buffer_to_vec, check, check_abi_version}; -use crate::writer::format_from_path; -use crate::{ - Date, Error, Time, Timestamp, WriteOptions, XlBuffer, XlWriterHandle, XL_T_BOOL, XL_T_DATE, - XL_T_F64, XL_T_I64, XL_T_TIME, XL_T_TIMESTAMP, -}; - -/// A streaming writer handle - the row-by-row counterpart to -/// [`Workbook`](crate::workbook::Workbook). +//! [`WriterHandle`] is the streaming counterpart to [`crate::write_columns`]/[`crate::write_sheet`]: +//! instead of materializing a whole table up front, it writes directly as each call arrives - one +//! sheet and one row open at a time. Call order mirrors the C ABI (see `xl_writer_handle` in +//! `excelreader.h`): `create`/`new`, then per sheet `start_sheet`..`end_sheet`, each containing +//! `start_row`..`end_row` with one `write_*` call per cell in between, left to right, then +//! `finish`. A call out of order returns an `Err` rather than corrupting output, and the handle +//! stays usable afterward - fix the call order and continue, or let `Drop` discard it. + +use crate::ffi::{self, XlWriterHandle}; +use crate::workbook::{buffer_to_vec, check, check_abi_version, len_i32, path_str}; +use crate::{ColumnType, Date, Error, Time, Timestamp, WriteOptions}; +use std::io::Write; +use std::path::Path; + +/// A streaming writer - the row-by-row counterpart to [`Workbook`](crate::Workbook). /// /// Not thread-safe - use one per thread, same contract as the C ABI. (The raw handle makes this /// type neither `Send` nor `Sync`, so the compiler enforces that for you.) /// -/// Dropping a `WriterHandle` closes and releases it (`xl_close_write_handle`), silently discarding -/// any error - same convention as [`Workbook`](crate::workbook::Workbook)'s `Drop`. Call -/// [`bytes`](Self::bytes) (memory-backed) or reopen the path (file-backed) to observe the actual -/// result; do not rely on the drop for that. -pub struct WriterHandle { +/// Call [`finish`](Self::finish) to complete the workbook and observe any error. Dropping a handle +/// without finishing it closes it on a best-effort basis, discarding any error, and writes nothing +/// to an `io::Write` destination. +pub struct WriterHandle<'w> { handle: *mut XlWriterHandle, + writer: Option>, } -impl WriterHandle { - /// Creates `path` (truncating it if it already exists), inferring the format from its - /// extension via [`crate::writer::format_from_path`]. - pub fn open(path: &str, options: Option<&WriteOptions>) -> Result { - Self::open_with(path, format_from_path(path), options) - } - - /// Creates `path` with an explicit format. `format` must be one of `XL_FORMAT_XLS`/`XLSX`/ - /// `XLSB`/`CSV` - [`XL_FORMAT_AUTO`](crate::XL_FORMAT_AUTO) is an error, the same as - /// [`write_columns`](crate::writer::write_columns). - pub fn open_with( - path: &str, - format: i32, - options: Option<&WriteOptions>, - ) -> Result { +impl WriterHandle<'static> { + /// Creates `path` (truncating it if it already exists). The format is + /// [`WriteOptions::format`], or inferred from the extension when that is + /// [`Format::Auto`](crate::Format::Auto). + pub fn create(path: impl AsRef, options: &WriteOptions) -> Result { check_abi_version()?; - let raw = options.map(WriteOptions::to_raw); - let raw_ptr = crate::options::ptr_or_null(&raw); + let path = path.as_ref(); + let format = options.format_for(Some(path))?; + let path = path_str(path)?; + let path_len = len_i32(path.len(), "path")?; + let raw = options.to_raw(); let mut handle: *mut XlWriterHandle = std::ptr::null_mut(); - let status = unsafe { - crate::xl_open_write_handle( - path.as_ptr(), - path.len() as i32, - format, - raw_ptr, - &mut handle, - ) - }; - check(status)?; - Ok(WriterHandle { handle }) - } - - /// In-memory equivalent of [`open_with`](Self::open_with): read the result back with - /// [`bytes`](Self::bytes). `format` is always required here - there is no path to infer one - /// from. - pub fn open_memory(format: i32, options: Option<&WriteOptions>) -> Result { + check(unsafe { + ffi::xl_open_write_handle(path.as_ptr(), path_len, format, &raw, &mut handle) + })?; + Ok(WriterHandle { + handle, + writer: None, + }) + } +} + +impl<'w> WriterHandle<'w> { + /// Builds the workbook in memory and writes it to `writer`, once, in [`finish`](Self::finish). + /// [`WriteOptions::format`] must be set - there is no extension to infer one from. + pub fn new(writer: impl Write + 'w, options: &WriteOptions) -> Result { check_abi_version()?; - let raw = options.map(WriteOptions::to_raw); - let raw_ptr = crate::options::ptr_or_null(&raw); + let format = options.format_for(None)?; + let raw = options.to_raw(); let mut handle: *mut XlWriterHandle = std::ptr::null_mut(); - let status = unsafe { crate::xl_open_write_handle_to_memory(format, raw_ptr, &mut handle) }; - check(status)?; - Ok(WriterHandle { handle }) + check(unsafe { ffi::xl_open_write_handle_to_memory(format, &raw, &mut handle) })?; + Ok(WriterHandle { + handle, + writer: Some(Box::new(writer)), + }) } /// Starts a new sheet named `name`. Must not be called again before the current sheet, if /// any, has been ended with [`end_sheet`](Self::end_sheet). pub fn start_sheet(&mut self, name: &str) -> Result<(), Error> { - check(unsafe { crate::xl_start_sheet(self.handle, name.as_ptr(), name.len() as i32) }) + check(unsafe { ffi::xl_start_sheet(self.handle, name.as_ptr(), name.len() as i32) }) } /// Starts a new row on the current sheet. Must not be called again before the current row, if /// any, has been ended with [`end_row`](Self::end_row). pub fn start_row(&mut self) -> Result<(), Error> { - check(unsafe { crate::xl_start_row(self.handle) }) + check(unsafe { ffi::xl_start_row(self.handle) }) } /// Writes the next cell of the current row as text, or a blank cell for `None`. @@ -92,14 +81,14 @@ impl WriterHandle { let (ptr, len) = value.map_or((std::ptr::null(), 0), |text| { (text.as_ptr(), text.len() as i32) }); - check(unsafe { crate::xl_write_string(self.handle, ptr, len) }) + check(unsafe { ffi::xl_write_string(self.handle, ptr, len) }) } /// Writes the next cell of the current row as an integer, or a blank cell for `None`. pub fn write_i64(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_int64(self.handle, v) }), - None => self.write_null(XL_T_I64), + Some(v) => check(unsafe { ffi::xl_write_int64(self.handle, v) }), + None => self.write_null(ColumnType::I64), } } @@ -107,86 +96,90 @@ impl WriterHandle { /// `None`. pub fn write_f64(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_float64(self.handle, v) }), - None => self.write_null(XL_T_F64), + Some(v) => check(unsafe { ffi::xl_write_float64(self.handle, v) }), + None => self.write_null(ColumnType::F64), } } /// Writes the next cell of the current row as a boolean, or a blank cell for `None`. pub fn write_bool(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_bool(self.handle, i32::from(v)) }), - None => self.write_null(XL_T_BOOL), + Some(v) => check(unsafe { ffi::xl_write_bool(self.handle, i32::from(v)) }), + None => self.write_null(ColumnType::Bool), } } /// Writes the next cell of the current row as a date, or a blank cell for `None`. pub fn write_date(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_date(self.handle, v.days_since_epoch) }), - None => self.write_null(XL_T_DATE), + Some(v) => check(unsafe { ffi::xl_write_date(self.handle, v.days_since_epoch) }), + None => self.write_null(ColumnType::Date), } } /// Writes the next cell of the current row as a time of day, or a blank cell for `None`. pub fn write_time(&mut self, value: Option