From c61888bacba80bfb5035884587a2d0ff4f387846 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 10:36:53 -0300 Subject: [PATCH 01/21] fix(python): tag wheels from the bundled library and build linux-arm64 Wheels took the build runner's own platform tag, so 6.0.0 shipped as macosx_26_0_arm64 and manylinux_2_39_x86_64 although the library needs only macOS 12.0 and GLIBC 2.34: pip refused it on macOS 14/15, Debian 12, Ubuntu 22.04, RHEL 9 and Amazon Linux 2023. The build hook now reads the floor from the library (LC_BUILD_VERSION minos, newest GLIBC_2.N symbol), and CI checks each wheel installs on the runner that built it. Adds ubuntu-24.04-arm to the native, CLI and wheel builds, and fixes the arch detection that labelled Linux aarch64 (uname -m) as x64. --- .github/workflows/native-bindings.yml | 10 ++++- .github/workflows/release.yml | 13 +++--- python/hatch_build.py | 57 +++++++++++++++++++-------- python/scripts/check_wheel_tag.py | 29 ++++++++++++++ 4 files changed, 85 insertions(+), 24 deletions(-) create mode 100644 python/scripts/check_wheel_tag.py diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 06d6862d..ea2b77ff 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -62,7 +62,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] steps: - name: Checkout @@ -172,7 +172,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] steps: - name: Checkout @@ -202,6 +202,12 @@ jobs: - name: Run the Python tests run: pytest python/tests -v + - name: Build the wheel and check it installs here + run: | + pip install build + python -m build --wheel python/ --outdir dist/ + python python/scripts/check_wheel_tag.py dist + - name: Build the C smoke test run: | cmake -S tests/ExcelReader.NativeSmoke -B build/nativesmoke -DCMAKE_BUILD_TYPE=Release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91fff678..65efa69d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,7 +67,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] steps: - name: Checkout @@ -92,7 +92,7 @@ jobs: macOS) src=ExcelReader.Native.dylib; os=osx; ext=dylib ;; *) src=ExcelReader.Native.so; os=linux; ext=so ;; esac - arch=$([ "$(uname -m 2>/dev/null || echo x64)" = "arm64" ] && echo arm64 || echo x64) + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; *) arch=x64 ;; esac mkdir -p release-assets cp "python/src/excelreader/_lib/${src}" "release-assets/excelreader-native-${os}-${arch}.${ext}" @@ -116,7 +116,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] steps: - name: Checkout @@ -133,7 +133,7 @@ jobs: id: rid shell: bash run: | - arch=$([ "$(uname -m 2>/dev/null || echo x64)" = "arm64" ] && echo arm64 || echo x64) + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; *) arch=x64 ;; esac case "${{ runner.os }}" in Windows) os=win ;; macOS) os=osx ;; @@ -175,7 +175,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] steps: - name: Checkout @@ -209,6 +209,9 @@ jobs: pip install build python -m build --wheel python/ --outdir dist/ + - name: Check the wheel installs here + run: python python/scripts/check_wheel_tag.py dist + - name: Upload wheel artifact uses: actions/upload-artifact@v7 with: diff --git a/python/hatch_build.py b/python/hatch_build.py index 9e4225a7..8214ad67 100644 --- a/python/hatch_build.py +++ b/python/hatch_build.py @@ -1,35 +1,58 @@ """Forces a platform-specific wheel tag: the wheel bundles a native library per OS (see _lib/), so it isn't pure Python even though every .py file in it is. The package has no CPython C-extension (it's ctypes calling a NativeAOT shared library), so it's -ABI/interpreter-independent - only the platform component of the tag should be specific.""" +ABI/interpreter-independent - only the platform component of the tag should be specific. + +The platform component comes from the bundled library, not from the build machine: a +runner's own tag (macosx_26_0, manylinux_2_39) shuts out every older system the library +actually loads on.""" import platform +import re +import struct +from pathlib import Path from hatchling.builders.hooks.plugin.interface import BuildHookInterface from packaging.tags import platform_tags +LIB_DIR = Path(__file__).parent / "src" / "excelreader" / "_lib" +LC_BUILD_VERSION = 0x32 + + +def glibc_floor(library: bytes) -> str: + """The newest GLIBC_2.N symbol version the library needs, as manylinux's `2_N`.""" + minors = [int(minor) for minor in re.findall(rb"GLIBC_2\.(\d+)", library)] + return f"2_{max(minors, default=17)}" + + +def macos_floor(library: bytes) -> str: + """The minimum macOS a thin 64-bit Mach-O declares in LC_BUILD_VERSION, as `MAJOR_0`.""" + commands = struct.unpack_from("> 16}_0" + offset += size + raise ValueError("the native library has no LC_BUILD_VERSION load command") -def _platform_tag() -> str: - """Picks a platform tag PyPI actually accepts. - packaging.tags.platform_tags() lists manylinux/musllinux-compatible tags first - and the bare `linux_` tag last - but on a runner where it can't confirm - manylinux compliance (no _manylinux metadata), the bare tag is all that's offered, - and PyPI rejects raw `linux_*`/`freebsd_*`/etc. wheel uploads outright. Skip that - bare tag and fall back to manylinux2014 (glibc 2.17, released 2012), a floor every - current Linux distro satisfies - there's nothing to detect here since this wheel - never links against glibc at build time (it dlopen()s a NativeAOT .so at runtime). - """ +def platform_tag() -> str: machine = platform.machine().lower() - bare_linux = f"linux_{machine}" - for tag in platform_tags(): - if tag != bare_linux: - return tag - return f"manylinux2014_{machine}" + system = platform.system() + if system == "Linux": + library = (LIB_DIR / "ExcelReader.Native.so").read_bytes() + return f"manylinux_{glibc_floor(library)}_{machine}" + if system == "Darwin": + library = (LIB_DIR / "ExcelReader.Native.dylib").read_bytes() + return f"macosx_{macos_floor(library)}_{machine}" + return next(iter(platform_tags())) class NativeLibraryBuildHook(BuildHookInterface): def initialize(self, version, build_data): build_data["pure_python"] = False build_data["infer_tag"] = True - build_data["tag"] = f"py3-none-{_platform_tag()}" + tag = next(iter(platform_tags())) if version == "editable" else platform_tag() + build_data["tag"] = f"py3-none-{tag}" diff --git a/python/scripts/check_wheel_tag.py b/python/scripts/check_wheel_tag.py new file mode 100644 index 00000000..d5874a01 --- /dev/null +++ b/python/scripts/check_wheel_tag.py @@ -0,0 +1,29 @@ +"""Fails unless every wheel in the given directory installs on this machine. + +Run after building a wheel: python python/scripts/check_wheel_tag.py dist +""" + +import sys +from pathlib import Path + +from packaging.tags import sys_tags +from packaging.utils import parse_wheel_filename + + +def main() -> int: + supported = set(sys_tags()) + wheels = sorted(Path(sys.argv[1]).glob("*.whl")) + if not wheels: + print(f"error: no wheel in {sys.argv[1]}", file=sys.stderr) + return 1 + for wheel in wheels: + tags = parse_wheel_filename(wheel.name)[3] + if tags.isdisjoint(supported): + print(f"error: {wheel.name} does not install on this machine", file=sys.stderr) + return 1 + print(f"ok: {wheel.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 86a636de7e612a7a8ae2d17080e2423b008c27c9 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 11:08:44 -0300 Subject: [PATCH 02/21] ci: report every required CodeQL check; build macOS x64 and musllinux wheels CodeQL picked its matrix from the languages a pull request touched, so a PR without C# changes never created "Analyze (csharp)" and the required check stayed pending. Both languages now always get a job, and an unchanged one skips its steps, as Build & Test already does. macos-15-intel joins the native, CLI and wheel builds. musl builds run in Alpine containers on x64 and arm64: build_native.py picks the linux-musl RID off glibc, and the wheel takes the musllinux tag when the library links musl. --- .github/workflows/codeql.yml | 38 ++++++++++------------ .github/workflows/native-bindings.yml | 21 ++++++++++-- .github/workflows/release.yml | 47 ++++++++++++++++++++++++--- python/hatch_build.py | 2 ++ python/scripts/build_musl_wheel.sh | 12 +++++++ python/scripts/build_native.py | 2 ++ 6 files changed, 95 insertions(+), 27 deletions(-) create mode 100644 python/scripts/build_musl_wheel.sh diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a0b08d88..269c03d5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,7 +24,8 @@ jobs: if: github.event.pull_request.draft != true runs-on: ubuntu-latest outputs: - languages: ${{ steps.languages.outputs.value }} + csharp: ${{ github.event_name != 'pull_request' || steps.filter.outputs.csharp == 'true' }} + python: ${{ github.event_name != 'pull_request' || steps.filter.outputs.python == 'true' }} steps: - name: Checkout uses: actions/checkout@v7 @@ -45,39 +46,32 @@ jobs: - 'python/**' - '.github/workflows/codeql.yml' - - name: Select languages - id: languages - run: | - if [ '${{ github.event_name }}' != 'pull_request' ]; then - echo 'value=["csharp","python"]' >> "$GITHUB_OUTPUT" - exit 0 - fi - languages=() - if [ '${{ steps.filter.outputs.csharp }}' = 'true' ]; then - languages+=('"csharp"') - fi - if [ '${{ steps.filter.outputs.python }}' = 'true' ]; then - languages+=('"python"') - fi - echo "value=[$(IFS=,; echo "${languages[*]}")]" >> "$GITHUB_OUTPUT" - + # Both languages always get a job: "Analyze (csharp)" and "Analyze (python)" are required checks, and + # a check whose job never starts stays pending. An unchanged language's job skips its steps instead. analyze: name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest needs: changes - if: needs.changes.outputs.languages != '[]' strategy: fail-fast: false matrix: - language: ${{ fromJSON(needs.changes.outputs.languages) }} + language: [csharp, python] + + env: + RUN: ${{ needs.changes.outputs[matrix.language] }} steps: + - name: Skip (language unchanged) + if: env.RUN != 'true' + run: echo "No ${{ matrix.language }} changes; skipping the analysis." + - name: Checkout + if: env.RUN == 'true' uses: actions/checkout@v7 - name: Setup .NET - if: matrix.language == 'csharp' + if: env.RUN == 'true' && matrix.language == 'csharp' uses: actions/setup-dotnet@v6 with: global-json-file: global.json @@ -85,6 +79,7 @@ jobs: cache-dependency-path: '**/*.csproj' - name: Initialize CodeQL + if: env.RUN == 'true' uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} @@ -93,13 +88,14 @@ jobs: # Only the shipped projects: tests, benchmarks and the fuzz harness double the build for code # no one runs against untrusted input. - name: Build (Release) - if: matrix.language == 'csharp' + if: env.RUN == 'true' && matrix.language == 'csharp' run: | for project in src/*/*.csproj; do dotnet build "$project" --configuration Release -p:RunAnalyzers=false done - name: Perform CodeQL Analysis + if: env.RUN == 'true' uses: github/codeql-action/analyze@v4 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index ea2b77ff..527b73a1 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -62,7 +62,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -172,7 +172,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -221,6 +221,23 @@ jobs: python python/benchmarks/bench_read.py --n 1 python python/benchmarks/bench_write.py --n 1 + test-musl: + name: musl library + Python (${{ matrix.runner }}) + needs: changes + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.python == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build, test and wheel in Alpine + run: sh python/scripts/build_musl_wheel.sh + test-cpp: name: C++ package + smoke test (${{ matrix.os }}) needs: [changes, build-native] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 65efa69d..1f367a03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,7 +67,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -116,7 +116,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -175,7 +175,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] steps: - name: Checkout @@ -219,9 +219,48 @@ jobs: path: dist/*.whl if-no-files-found: error + musl: + name: Build musl library and wheel (${{ matrix.arch }}) + needs: build + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - { arch: x64, runner: ubuntu-latest } + - { arch: arm64, runner: ubuntu-24.04-arm } + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set package version + run: sed -i "s/^version = \".*\"/version = \"${{ needs.build.outputs.version }}\"/" python/pyproject.toml + + - name: Build, test and wheel in Alpine + run: sh python/scripts/build_musl_wheel.sh + + - name: Rename asset for this platform + run: | + mkdir -p release-assets + cp python/src/excelreader/_lib/ExcelReader.Native.so "release-assets/excelreader-native-linux-musl-${{ matrix.arch }}.so" + + - name: Upload release asset + uses: actions/upload-artifact@v7 + with: + name: release-native-musl-${{ matrix.arch }} + path: release-assets/* + if-no-files-found: error + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: wheel-musl-${{ matrix.arch }} + path: dist/*.whl + if-no-files-found: error + publish: name: Publish to NuGet & GitHub Release - needs: [build, native, cli, python] + needs: [build, native, cli, python, musl] runs-on: ubuntu-latest permissions: contents: write diff --git a/python/hatch_build.py b/python/hatch_build.py index 8214ad67..c74d27b3 100644 --- a/python/hatch_build.py +++ b/python/hatch_build.py @@ -43,6 +43,8 @@ def platform_tag() -> str: system = platform.system() if system == "Linux": library = (LIB_DIR / "ExcelReader.Native.so").read_bytes() + if b"libc.musl-" in library: + return next(tag for tag in platform_tags() if tag.startswith("musllinux_")) return f"manylinux_{glibc_floor(library)}_{machine}" if system == "Darwin": library = (LIB_DIR / "ExcelReader.Native.dylib").read_bytes() diff --git a/python/scripts/build_musl_wheel.sh b/python/scripts/build_musl_wheel.sh new file mode 100644 index 00000000..95448cb2 --- /dev/null +++ b/python/scripts/build_musl_wheel.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Builds the linux-musl native library and its musllinux wheel in Alpine containers, running the +# Python tests against it. Needs a Linux host with Docker; the wheel lands in dist/. +# +# Run from the repository root: sh python/scripts/build_musl_wheel.sh +set -eu + +docker run --rm -v "$PWD:/repo" -w /repo mcr.microsoft.com/dotnet/sdk:10.0-alpine \ + sh -c 'apk add --no-cache clang build-base zlib-dev python3 && python3 python/scripts/build_native.py' + +docker run --rm -v "$PWD:/repo" -w /repo python:3.12-alpine \ + sh -c 'pip install -e "python[dev]" build && pytest python/tests && python -m build --wheel python/ --outdir dist/ && python python/scripts/check_wheel_tag.py dist' diff --git a/python/scripts/build_native.py b/python/scripts/build_native.py index c85cf58d..319a3437 100644 --- a/python/scripts/build_native.py +++ b/python/scripts/build_native.py @@ -30,6 +30,8 @@ def default_rid() -> str: return f"win-{arch}" if system == "Darwin": return f"osx-{arch}" + if platform.libc_ver()[0] != "glibc": + return f"linux-musl-{arch}" return f"linux-{arch}" From 44311c9cf20c9de32a25097bba667f253b6ae401 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 13:12:03 -0300 Subject: [PATCH 03/21] docs: state the 7.x compatibility policy and validate packages against the last release A versioning guide says what a minor version may change: new members on the interfaces only the library implements (with default implementations), new enum values, new options. The interfaces users implement gain nothing they would have to implement. Those library-implemented interfaces and the two growing enums now say so in their docs, and CONTRIBUTING lists the rules. ExcelReader.NET and ExcelReader.Arrow enable package validation against 7.0.0-rc.2, and CI packs both on every pull request, so a removed or changed member fails the build. The development version derives from the baseline, since validation requires an assembly version at least the baseline's; release builds still override it from the tag. --- .github/workflows/ci.yml | 7 ++ CONTRIBUTING.md | 15 ++++ docs/guide/versioning.md | 86 +++++++++++++++++++ docs/snippets/Versioning.cs | 20 +++++ docs/toc.yml | 2 + .../ExcelReader.Arrow.csproj | 8 +- src/ExcelReader.Core/ExcelReader.Core.csproj | 8 +- src/ExcelReader.Core/Reader/CellType.cs | 5 +- .../Reader/IExcelRowEnumerator.cs | 1 + src/ExcelReader.Core/Reader/IExcelWorkbook.cs | 7 +- .../Reader/Schema/ExcelColumnType.cs | 1 + .../Writer/ITableSheetWriter.cs | 1 + .../Writer/IWorkbookWriter.cs | 3 + 13 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 docs/guide/versioning.md create mode 100644 docs/snippets/Versioning.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23f980d8..af437c51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,13 @@ jobs: if: env.RUN == 'true' && matrix.os == 'ubuntu-latest' run: dotnet format whitespace ExcelReader.slnx --verify-no-changes + # Packing runs package validation, which fails on any binary break against the last release. + - name: Validate packages against the last release + if: env.RUN == 'true' && matrix.os == 'ubuntu-latest' + run: | + dotnet pack src/ExcelReader.Core/ExcelReader.Core.csproj --configuration Release --no-build --output ./package-validation + dotnet pack src/ExcelReader.Arrow/ExcelReader.Arrow.csproj --configuration Release --no-build --output ./package-validation + - name: Test (Release) if: env.RUN == 'true' run: >- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82fea814..b3150733 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,21 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas `src/ExcelReader.Core/PublicAPI/PublicAPI.Unshipped.txt`. A bot promotes `Unshipped` → `Shipped` automatically after each release — don't edit `Shipped.txt` by hand. +- **Minor versions don't break callers.** [docs/guide/versioning.md](docs/guide/versioning.md) is the + policy; in practice: + - A new member on a library-implemented interface (`IExcelWorkbook`, `IExcelSheet`, + `IExcelRowEnumerator`, `IWorkbookWriter`, `ISheetWriter`, `ITableSheetWriter`, `IRowWriter`) + gets a default implementation. A user-implemented one (`IExcelCellConverter`, `IExcelCellWriter`, + `IExcelRowMap`, `IExcelRecordMap`, `IRowRecord`, `IRowAccumulator`) gets no new member a user + would have to implement. + - A feature only some formats support goes on the concrete types first, or on its own interface + the way `ITableSheetWriter` sits beside `ISheetWriter`. + - New inputs go on the options object or a new overload, never as a new optional parameter on an + existing method. + - `dotnet pack` runs package validation against `PackageValidationBaselineVersion` in the + `ExcelReader.Core` and `ExcelReader.Arrow` projects, and CI runs it on every pull request. + After each release, move the baseline to the version just published. + - **Tests are required for behavior changes.** Run the suite before opening a PR: ```bash diff --git a/docs/guide/versioning.md b/docs/guide/versioning.md new file mode 100644 index 00000000..a4a15214 --- /dev/null +++ b/docs/guide/versioning.md @@ -0,0 +1,86 @@ +# Versioning and compatibility + +What a new version can change, and how to write code that keeps compiling and running across minor +versions. The .NET packages follow [Semantic Versioning](https://semver.org/): a major version (7.0, +8.0) may break your code, while a minor (7.1) or a patch (7.0.1) does not. + +## What a minor version may add + +- New types, new members on existing types, and new overloads. +- New properties on the options objects, such as + [`ExcelReaderOptions`](xref:ExcelReader.Core.Reader.ExcelReaderOptions). A new option always defaults + to the previous behavior. +- New members on the interfaces that only this library implements (below). +- New values in the enums (below). +- Bug fixes. A file that used to read wrong, or a written file that Excel refused or altered, can come + out differently after a fix. Other changes in what an existing call returns or throws wait for a + major version and are listed in its migration guide. + +A minor version does not remove or rename anything, change an existing signature, or add a +parameter to an existing method. + +## Interfaces you implement + +These are the extension points. A minor version never adds a member to them that you would have to +implement: + +- [`IExcelCellConverter`](xref:ExcelReader.Core.Parser.IExcelCellConverter`1) and + [`IExcelCellWriter`](xref:ExcelReader.Core.Writer.IExcelCellWriter`1) +- [`IExcelRowMap`](xref:ExcelReader.Core.Parser.IExcelRowMap`1) and + [`IExcelRecordMap`](xref:ExcelReader.Core.Writer.IExcelRecordMap`1) +- [`IRowRecord`](xref:ExcelReader.Core.Reader.IRowRecord`1) and + [`IRowAccumulator`](xref:ExcelReader.Core.Reader.IRowAccumulator`2) + +New information reaches them through the types they already receive. For example, a converter gets +it as a new member of [`ExcelRowContext`](xref:ExcelReader.Core.Parser.ExcelRowContext), not as a +new parameter. + +## Interfaces only the library implements + +The reading and writing interfaces describe what every format's workbook, sheet and row writer can +do. Program against them, but do not implement them: + +- [`IExcelWorkbook`](xref:ExcelReader.Core.Reader.IExcelWorkbook), + [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet) and + [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet`1) +- [`IExcelRowEnumerator`](xref:ExcelReader.Core.Reader.IExcelRowEnumerator) +- [`IWorkbookWriter`](xref:ExcelReader.Core.Writer.IWorkbookWriter`1), + [`ISheetWriter`](xref:ExcelReader.Core.Writer.ISheetWriter`1), + [`ITableSheetWriter`](xref:ExcelReader.Core.Writer.ITableSheetWriter`1) and + [`IRowWriter`](xref:ExcelReader.Core.Writer.IRowWriter) + +A minor version can add members to them. Each new member comes with a default implementation, so an +existing implementation still compiles and loads, but it throws `NotSupportedException` or falls back +to a slower path for the new member. A feature that only some formats support appears first on the +concrete types (`XlsxSheetWriter`, `XlsbWorkbook`), or as a separate interface the way +[`ITableSheetWriter`](xref:ExcelReader.Core.Writer.ITableSheetWriter`1) sits beside +[`ISheetWriter`](xref:ExcelReader.Core.Writer.ISheetWriter`1). + +## Enums that grow + +[`CellType`](xref:ExcelReader.Core.Reader.CellType) and +[`ExcelColumnType`](xref:ExcelReader.Core.Reader.Schema.ExcelColumnType) can gain values in a minor +version, as can their native counterparts `XL_CELL_*` and `XL_T_*`. An existing value never changes +its number or meaning. A `switch` over them should have a default branch: + + +```csharp +string text = cell.Type switch +{ + CellType.Empty => "", + CellType.Number when cell.TryGetDouble(out double number) => number.ToString(CultureInfo.InvariantCulture), + _ => cell.GetString(), +}; +``` + +## The native library and the bindings + +The C ABI is versioned on its own by `XL_ABI_VERSION`, which `xl_abi_version()` reports. It changes +only when a signature or a struct layout does: 7.0 kept version 6. Every options struct starts with +`struct_size`, so a field added in a later version does not break a caller compiled against an +earlier header. The Rust, Python and C++ bindings check the ABI version when they load the library. + +## How it is enforced + +Every pull request packs `ExcelReader.NET` and `ExcelReader.Arrow` and compares them with the last +release through .NET package validation. A removed or changed member fails the build. diff --git a/docs/snippets/Versioning.cs b/docs/snippets/Versioning.cs new file mode 100644 index 00000000..76c6af3a --- /dev/null +++ b/docs/snippets/Versioning.cs @@ -0,0 +1,20 @@ +using System.Globalization; + +namespace Snippets.Versioning.EnumSwitch +{ + internal static class Snippet + { + internal static string Run(Cell cell) + { + #region versioning-enum-switch + string text = cell.Type switch + { + CellType.Empty => "", + CellType.Number when cell.TryGetDouble(out double number) => number.ToString(CultureInfo.InvariantCulture), + _ => cell.GetString(), + }; + #endregion + return text; + } + } +} diff --git a/docs/toc.yml b/docs/toc.yml index d3707a7d..62aad411 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -14,6 +14,8 @@ href: guide/errors.md - name: Reading untrusted files href: guide/untrusted-input.md + - name: Versioning and compatibility + href: guide/versioning.md - name: API href: api/ - name: Bindings diff --git a/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj b/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj index 2e0e098e..db16a3c1 100644 --- a/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj +++ b/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj @@ -5,9 +5,15 @@ true + + + true + 7.0.0-rc.2 + + ExcelReader.Arrow - 1.0.0 + $(PackageValidationBaselineVersion)-dev Gabriel Matte ExcelReader.Arrow Apache Arrow RecordBatch conversion for ExcelReader.NET. diff --git a/src/ExcelReader.Core/ExcelReader.Core.csproj b/src/ExcelReader.Core/ExcelReader.Core.csproj index 3bf3c80e..8bfd9514 100644 --- a/src/ExcelReader.Core/ExcelReader.Core.csproj +++ b/src/ExcelReader.Core/ExcelReader.Core.csproj @@ -8,10 +8,16 @@ true + + + true + 7.0.0-rc.2 + + ExcelReader.NET - 1.0.0 + $(PackageValidationBaselineVersion)-dev Gabriel Matte ExcelReader High-performance, low-allocation .NET library for reading and writing XLSX, XLSB, XLS, and CSV — streaming readers, typed row parsing, and workbook writers for all four formats. diff --git a/src/ExcelReader.Core/Reader/CellType.cs b/src/ExcelReader.Core/Reader/CellType.cs index d770a84d..d901cea4 100644 --- a/src/ExcelReader.Core/Reader/CellType.cs +++ b/src/ExcelReader.Core/Reader/CellType.cs @@ -5,7 +5,10 @@ namespace ExcelReader.Core.Reader /// /// Identifies the kind of value a holds. /// - /// A formula cell reports the type of its cached result, the same in every format. + /// + /// A formula cell reports the type of its cached result, the same in every format. + /// A minor version can add values, so a over this enum should have a default branch. See versioning and compatibility. + /// [SuppressMessage("Design", "CA1027:Mark enums with FlagsAttribute", Justification = "Not flags: the values match the native XL_CELL_* codes, and 5 belonged to the removed Formula.")] public enum CellType { diff --git a/src/ExcelReader.Core/Reader/IExcelRowEnumerator.cs b/src/ExcelReader.Core/Reader/IExcelRowEnumerator.cs index cda5df19..f49e7b04 100644 --- a/src/ExcelReader.Core/Reader/IExcelRowEnumerator.cs +++ b/src/ExcelReader.Core/Reader/IExcelRowEnumerator.cs @@ -5,6 +5,7 @@ namespace ExcelReader.Core.Reader /// enumerator and driven either synchronously () or asynchronously /// (). /// + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. public interface IExcelRowEnumerator : IDisposable, IAsyncDisposable { /// Gets the row at the enumerator's current position. Only valid after a call to or has returned . diff --git a/src/ExcelReader.Core/Reader/IExcelWorkbook.cs b/src/ExcelReader.Core/Reader/IExcelWorkbook.cs index ce11d2be..5b4f6964 100644 --- a/src/ExcelReader.Core/Reader/IExcelWorkbook.cs +++ b/src/ExcelReader.Core/Reader/IExcelWorkbook.cs @@ -19,6 +19,7 @@ namespace ExcelReader.Core.Reader /// Disposing the workbook closes it to new sheets and enumerators. Its file, stream and pooled /// buffers are released when the last enumerator obtained from it is disposed. /// + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. /// public interface IExcelWorkbook : IDisposable, IAsyncDisposable { @@ -66,6 +67,7 @@ public interface IExcelWorkbook : IDisposable, IAsyncDisposable /// . /// /// The concrete row enumerator type this sheet produces. + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. public interface IExcelSheet where TEnumerator : IExcelRowEnumerator { @@ -86,7 +88,10 @@ public interface IExcelSheet } /// A sheet of an : its position, name and visibility, and its rows. - /// A sheet holds no resources. Every call to starts an independent read. + /// + /// A sheet holds no resources. Every call to starts an independent read. + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. + /// public interface IExcelSheet : IExcelSheet { /// Gets the sheet's zero-based position in its workbook. diff --git a/src/ExcelReader.Core/Reader/Schema/ExcelColumnType.cs b/src/ExcelReader.Core/Reader/Schema/ExcelColumnType.cs index d22685a9..0d31464e 100644 --- a/src/ExcelReader.Core/Reader/Schema/ExcelColumnType.cs +++ b/src/ExcelReader.Core/Reader/Schema/ExcelColumnType.cs @@ -8,6 +8,7 @@ namespace ExcelReader.Core.Reader.Schema /// The underlying values are fixed: they are the XL_T_* constants of the native C ABI /// (see src/ExcelReader.Native/include/excelreader.h), which marshals this enum with a /// plain integer cast rather than a translation table. Renumbering a member is a silent ABI break. + /// A minor version can add values, so a over this enum should have a default branch. See versioning and compatibility. /// public enum ExcelColumnType { diff --git a/src/ExcelReader.Core/Writer/ITableSheetWriter.cs b/src/ExcelReader.Core/Writer/ITableSheetWriter.cs index d036d8e0..091df23d 100644 --- a/src/ExcelReader.Core/Writer/ITableSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/ITableSheetWriter.cs @@ -7,6 +7,7 @@ namespace ExcelReader.Core.Writer /// XLSX and XLSB sheet writers. /// /// The concrete row writer type. + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. public interface ITableSheetWriter : ISheetWriter { /// diff --git a/src/ExcelReader.Core/Writer/IWorkbookWriter.cs b/src/ExcelReader.Core/Writer/IWorkbookWriter.cs index f39559bf..bea1c0b2 100644 --- a/src/ExcelReader.Core/Writer/IWorkbookWriter.cs +++ b/src/ExcelReader.Core/Writer/IWorkbookWriter.cs @@ -15,6 +15,7 @@ namespace ExcelReader.Core.Writer /// instances it hands out. All of them carry mutable state (current /// sheet/row, buffered output) with no synchronization. Use one workbook writer per thread; do not /// call into a writer, its current sheet, or its current row from more than one thread at a time. + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. /// public interface IWorkbookWriter : IDisposable, IAsyncDisposable { @@ -90,6 +91,7 @@ public interface IWorkbookWriter : IDisposable, IAsyncDisposable /// for a sheet with no rows. /// /// The concrete this sheet produces. + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. public interface ISheetWriter : IDisposable, IAsyncDisposable { /// @@ -176,6 +178,7 @@ public interface ISheetWriter : IDisposable, IAsyncDisposable /// generic overload) still advances the column position by one, but whether it emits an explicit /// blank cell or omits a cell record entirely for that column is implementation-defined — either /// way, reading the cell back yields an empty/blank value. + /// Only ExcelReader implements this interface, and a minor version can add members to it. See versioning and compatibility. /// public interface IRowWriter : IDisposable, IAsyncDisposable { From a5b476d83285daedc6b50967162f7da6c6d44372 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 15:55:34 -0300 Subject: [PATCH 04/21] feat(python)!: open every kind of input through open_workbook open_workbook() now takes a path, bytes, bytearray, memoryview, io.BytesIO, a Source, or any object with readinto()/read(), and picks the native open from the input's type. open_bytes(), open_source() and open_stream() are removed; the last two only existed in the 7.0 release candidates. BREAKING CHANGE: open_bytes(data, ...) becomes open_workbook(data, ...). --- docs/guide/migrating-to-v7.md | 9 +++ python/README.md | 38 ++++++------- python/src/excelreader/__init__.py | 7 +-- python/src/excelreader/reader.py | 85 +++++++++++++++++++--------- python/src/excelreader/sources.py | 28 +++------ python/src/excelreader/types.py | 13 +++-- python/src/excelreader/writer.py | 2 +- python/tests/test_reader.py | 7 +-- python/tests/test_sheets.py | 32 +++++------ python/tests/test_sources.py | 91 +++++++++++++++++++++--------- python/tests/test_writer.py | 5 +- python/tests/test_writer_handle.py | 8 +-- 12 files changed, 190 insertions(+), 135 deletions(-) diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md index 555605d6..7e453286 100644 --- a/docs/guide/migrating-to-v7.md +++ b/docs/guide/migrating-to-v7.md @@ -160,3 +160,12 @@ The C ABI is unchanged: no signature moved, and `XL_ABI_VERSION` stays 6. Behavi `xl_write_options.date1904` now applies to XLSX as well as XLSB and XLS; v6 ignored it for XLSX. `xl_csv_aggregate` keeps its signature. Inside, it opens a [`CsvReader`](xref:ExcelReader.Core.Reader.Csv.CsvReader) with the dialect from the native struct and calls `SheetParallel`. + +### Python: one way to open a workbook + +`open_bytes()` is removed. `open_workbook()` takes the bytes instead, and also accepts a `bytearray`, a `memoryview` or an `io.BytesIO`. It also takes a `Source` or a readable stream, the inputs that 7.0's release candidates opened with `open_source()` and `open_stream()`, which are gone as well. Arguments and keywords are unchanged. + +| v6 | v7 | +|---|---| +| `open_bytes(data, "xlsx", password=p)` | `open_workbook(data, "xlsx", password=p)` | +| `open_bytes(buffer.getvalue())` | `open_workbook(buffer)` | diff --git a/python/README.md b/python/README.md index d694c42f..476d9f2d 100644 --- a/python/README.md +++ b/python/README.md @@ -378,25 +378,25 @@ never held twice, once as Arrow and once as the DataFrame. `ExcelReaderError` — the failure crosses the Arrow C Data Interface before this library's own exception wrapping ever runs. -### From memory +### From memory, a stream, or your own bytes -```python -from excelreader import open_bytes +`open_workbook()` takes more than a path. `bytes`, `bytearray`, `memoryview` and `io.BytesIO` are +read from memory; the format comes from the content, so pass `format` only for a CSV or to skip the +check. -with open_bytes(payload) as workbook: +```python +with open_workbook(payload) as workbook: ... ``` -### From your own bytes - -`open_source()` takes any object with a `size` and a `read_at(offset, buffer)` that fills `buffer` -and returns the count. The library asks for 4 MiB blocks and caches them, and every sheet can still -be read on its own thread. +An object with a `size` and a `read_at(offset, buffer)` that fills `buffer` and returns the count is +read in blocks on demand: the library asks for 4 MiB blocks and caches them, and every sheet can +still be read on its own thread. ```python import urllib.request -from excelreader import open_source +from excelreader import open_workbook class HttpSource: def __init__(self, url): @@ -411,28 +411,28 @@ class HttpSource: buffer[: len(data)] = data return len(data) -with open_source(HttpSource("https://example.com/book.xlsx")) as workbook: +with open_workbook(HttpSource("https://example.com/book.xlsx")) as workbook: ... ``` -`open_stream()` reads anything with `readinto()` or `read()`, such as `sys.stdin.buffer`. A CSV is -read as it arrives, and its first 16 MiB are kept so that `infer_schema()` followed by a read works; +Any other object with `readinto()` or `read()`, such as an open file or `sys.stdin.buffer`, is read +as a stream. A CSV is read as it arrives, and its first 16 MiB are kept so that `infer_schema()` followed by a read works; an XLSX, XLSB or XLS is read whole first. ```python import sys -from excelreader import open_stream +from excelreader import open_workbook -with open_stream(sys.stdin.buffer, "csv") as workbook: +with open_workbook(sys.stdin.buffer, "csv") as workbook: ... ``` -Both callbacks run with the GIL held. For a file on disk, `open_workbook()` is faster. +Both callbacks run with the GIL held. For a file on disk, pass its path: that is faster. ### Reader options -`open_workbook()`/`open_bytes()` take an optional `OpenOptions` for CSV dialect settings and reader +`open_workbook()` takes an optional `OpenOptions` for CSV dialect settings and reader resource limits. Every field defaults to `None`, meaning "use the library default", so you set only what you want to change. @@ -469,7 +469,7 @@ check off, for a file whose producer wrote wrong checksums. ### Encrypted workbooks -`open_workbook()`/`open_bytes()` take a `password` keyword to open a password-protected OOXML +`open_workbook()` takes a `password` keyword to open a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm): ```python @@ -504,7 +504,7 @@ encrypt_package("plain.xlsx", "secret.xlsx", "hunter2") `package_path` is read twice (it is not disposed or removed), so it must already be a finished file. Encryption parameters are fixed at Excel's own defaults — there are no options — and only XLSX/XLSB -packages can be encrypted, matching what `open_workbook`/`open_bytes` can decrypt. +packages can be encrypted, matching what `open_workbook` can decrypt. `encrypt_package_bytes()` does the same over bytes, so the plaintext package never has to be written to disk: diff --git a/python/src/excelreader/__init__.py b/python/src/excelreader/__init__.py index 6b8457d1..0d54a7e3 100644 --- a/python/src/excelreader/__init__.py +++ b/python/src/excelreader/__init__.py @@ -1,8 +1,8 @@ """Python bindings for ExcelReader.""" from excelreader.arrow_stream import ArrowStream -from excelreader.reader import Sheet, Sheets, Workbook, decode_cell, open_bytes, open_workbook -from excelreader.sources import Source, open_source, open_stream +from excelreader.reader import Sheet, Sheets, Workbook, decode_cell, open_workbook +from excelreader.sources import Source from excelreader.types import ( Cell, CellType, @@ -55,9 +55,6 @@ "decode_cell", "encrypt_package", "encrypt_package_bytes", - "open_bytes", - "open_source", - "open_stream", "open_workbook", "open_writer", "open_writer_to_memory", diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index ca87f8ef..0790a2b4 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -3,17 +3,21 @@ from __future__ import annotations import ctypes +import io import operator +import os import struct import threading from array import array from collections.abc import Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, SupportsIndex, overload +from typing import IO, TYPE_CHECKING, SupportsIndex, overload if TYPE_CHECKING: from typing_extensions import Self + from excelreader.sources import Source + from excelreader import _native from excelreader.arrow_stream import ArrowStream from excelreader.types import ( @@ -711,43 +715,68 @@ def _raw_options( def open_workbook( - path: str | Path, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None + source: str | os.PathLike[str] | bytes | bytearray | memoryview | IO[bytes] | Source, + format: str | None = None, + *, + options: OpenOptions | None = None, + password: str | None = None, ) -> Workbook: - """Opens a workbook from disk. `format` is one of auto/xls/xlsx/xlsb/csv; None infers it. - - `options` overrides reader limits and CSV dialect settings; see `OpenOptions`. `password` unlocks - an encrypted OOXML workbook (.xlsx/.xlsb); omitting it for one raises `PasswordRequiredError`, and - a wrong one raises `PasswordIncorrectError`. An explicit xlsx/xlsb `format` works for an encrypted - file too, the same as leaving `format` unset — both routes decrypt correctly given the right - `password`. + """Opens a workbook from a path, bytes in memory, a `Source`, or a readable stream. + + `source` picks how the workbook is read: + + - a `str` or path-like: the file on disk. + - `bytes`, `bytearray`, `memoryview` or `io.BytesIO`: a copy the native side takes immediately. + A `BytesIO` is read whole, whatever its position. + - a `Source` (`size` and `read_at`): fetched in blocks on demand, and every sheet can be read on + its own thread. The library keeps a reference to it until the workbook and everything read + from it are closed. `read_at` runs with the GIL held, so a local file is better opened by path. + - any other object with `readinto()` or `read()`, such as an open file, a pipe or + `sys.stdin.buffer`: read as a stream that cannot seek. A CSV is read as it arrives, and can be + read again from the start (`infer_schema()`, then a read) until 16 MiB of it were read; an + XLSX, XLSB or XLS is read whole before the workbook opens. The library never closes it. + + `format` is one of auto/xls/xlsx/xlsb/csv; None infers it, from the extension for a path and from + the content otherwise. The content tells XLS, XLSX and XLSB apart, and a stream that is none of + them reads as CSV; a CSV in memory or behind a `Source` needs `format="csv"`. `options` overrides reader limits and CSV dialect settings; see + `OpenOptions`. `password` unlocks an encrypted OOXML workbook (.xlsx/.xlsb); omitting it for one + raises `PasswordRequiredError`, and a wrong one raises `PasswordIncorrectError`. """ - resolved = Path(path) - encoded = str(resolved).encode("utf-8") + if isinstance(source, (str, os.PathLike)): + return _open_path(Path(source), format, options, password) + if isinstance(source, io.BytesIO): + source = source.getvalue() + if isinstance(source, (bytes, bytearray, memoryview)): + return _open_memory(source, format, options, password) + + from excelreader import sources + + if hasattr(source, "read_at"): + return sources._open_source(source, format, options, password) + if hasattr(source, "readinto") or hasattr(source, "read"): + return sources._open_stream(source, format, options, password) + raise TypeError( + "open_workbook() needs a path, bytes, io.BytesIO, a Source, or an object with readinto() or read();" + f" got {type(source).__name__}" + ) + + +def _open_path(path: Path, format: str | None, options: OpenOptions | None, password: str | None) -> Workbook: + encoded = str(path).encode("utf-8") handle = ctypes.c_void_p() lib = _native.load_library() raw_options, _raw, _password_buffer = _raw_options(options, password) - _check( - lib.xl_open_file( - encoded, len(encoded), _resolve_format(format, resolved), raw_options, ctypes.byref(handle) - ) - ) + _check(lib.xl_open_file(encoded, len(encoded), _resolve_format(format, path), raw_options, ctypes.byref(handle))) return Workbook(handle) -def open_bytes( - data: bytes, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +def _open_memory( + data: bytes | bytearray | memoryview, format: str | None, options: OpenOptions | None, password: str | None ) -> Workbook: - """Opens a workbook from an in-memory buffer. The native side copies `data` immediately. - - `options` overrides reader limits and CSV dialect settings; see `OpenOptions`. `password` unlocks - an encrypted OOXML workbook — see `open_workbook()` for the full contract. - """ + if not isinstance(data, bytes): + data = bytes(data) handle = ctypes.c_void_p() lib = _native.load_library() raw_options, _raw, _password_buffer = _raw_options(options, password) - _check( - lib.xl_open_memory( - data, len(data), _resolve_format(format, None), raw_options, ctypes.byref(handle) - ) - ) + _check(lib.xl_open_memory(data, len(data), _resolve_format(format, None), raw_options, ctypes.byref(handle))) return Workbook(handle) diff --git a/python/src/excelreader/sources.py b/python/src/excelreader/sources.py index faf39835..701be0f0 100644 --- a/python/src/excelreader/sources.py +++ b/python/src/excelreader/sources.py @@ -1,4 +1,4 @@ -"""Open a workbook from bytes the caller serves: a positional `Source`, or a readable stream.""" +"""Reading a workbook through callbacks: a positional `Source`, or a readable stream.""" from __future__ import annotations @@ -13,7 +13,7 @@ class Source(Protocol): - """Random-access bytes for `open_source()`. `read_at` is called from several threads at once.""" + """Random-access bytes for `open_workbook()`. `read_at` is called from several threads at once.""" size: int @@ -60,7 +60,7 @@ def _read(key: int, buffer: int, length: int) -> int: return len(data) count = readinto(_view(buffer, length)) if count is None: - raise OSError("the stream has no data ready; open_stream() needs a blocking stream") + raise OSError("the stream has no data ready; open_workbook() needs a blocking stream") count = operator.index(count) if count > length: raise OSError("the stream returned more bytes than requested") @@ -74,14 +74,9 @@ def _release(key: int) -> None: _live.pop(key, None) -def open_source( - source: Source, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +def _open_source( + source: Source, format: str | None, options: OpenOptions | None, password: str | None ) -> Workbook: - """Opens a workbook over bytes `source` serves; every sheet can be read on its own thread. - - The library keeps a reference to `source` until the workbook and everything read from it are - closed. `read_at` runs with the GIL held, so a local file is better opened with `open_workbook()`. - """ size = operator.index(source.size) if size < 0: raise ValueError("source.size must not be negative") @@ -96,18 +91,9 @@ def open_source( return Workbook(handle) -def open_stream( - stream: object, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +def _open_stream( + stream: object, format: str | None, options: OpenOptions | None, password: str | None ) -> Workbook: - """Opens a workbook over a stream that cannot seek, such as `sys.stdin.buffer` or a socket. - - A CSV (`format="csv"`, or no format and neither a ZIP nor an OLE2 signature) is read as it - arrives, and can be read again from the start - (`infer_schema()`, then a read) until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole - before the workbook opens. `stream` needs `readinto` or `read`; the library never closes it. - """ - if not (hasattr(stream, "readinto") or hasattr(stream, "read")): - raise TypeError("open_stream() needs an object with readinto() or read()") lib = _native.load_library() resolved = _resolve_format(format, None) raw_options, _raw, _password_buffer = _raw_options(options, password) diff --git a/python/src/excelreader/types.py b/python/src/excelreader/types.py index 5a143485..c41eaf97 100644 --- a/python/src/excelreader/types.py +++ b/python/src/excelreader/types.py @@ -119,7 +119,7 @@ class ColumnSpec(NamedTuple): class OpenOptions(NamedTuple): - """Reader limits and dialect settings for `open_workbook()`/`open_bytes()`. + """Reader limits and dialect settings for `open_workbook()`. Every field is None for "use the library default", so you only set what you actually want to override. That single convention replaces two the C ABI uses internally: numeric fields there @@ -131,9 +131,10 @@ class OpenOptions(NamedTuple): `csv_delimiter` and `csv_quote` are byte values, not strings: pass `ord(';')`, not `';'`. - `source_block_size` and `source_cache_bytes` tune the block cache of `open_source()` (4 MiB blocks - in 64 MiB by default; a negative block size turns the cache off). `max_buffered_bytes` caps how - much `open_stream()` reads before an XLSX, XLSB or XLS opens. + `source_block_size` and `source_cache_bytes` tune the block cache of a `Source` passed to + `open_workbook()` (4 MiB blocks in 64 MiB by default; a negative block size turns the cache off). + `max_buffered_bytes` caps how much `open_workbook()` reads from a stream before an XLSX, XLSB or + XLS opens. `verify_zip_checksums` (on by default) checks each ZIP entry of an XLSX or XLSB against its CRC-32, so a damaged file raises instead of returning wrong cells. Set it to `False` to read a @@ -249,10 +250,10 @@ class ExcelReaderError(Exception): class PasswordRequiredError(ExcelReaderError): - """Raised by `open_workbook()`/`open_bytes()` when the workbook is encrypted and no `password` + """Raised by `open_workbook()` when the workbook is encrypted and no `password` was supplied. Maps XL_STATUS_PASSWORD_REQUIRED from the native ABI.""" class PasswordIncorrectError(ExcelReaderError): - """Raised by `open_workbook()`/`open_bytes()` when the supplied `password` did not match the + """Raised by `open_workbook()` when the supplied `password` did not match the workbook's verifier. Maps XL_STATUS_PASSWORD_INCORRECT from the native ABI.""" diff --git a/python/src/excelreader/writer.py b/python/src/excelreader/writer.py index a9ea5db4..abc4196e 100644 --- a/python/src/excelreader/writer.py +++ b/python/src/excelreader/writer.py @@ -354,7 +354,7 @@ def write_polars_to_bytes(df: Any, *, format: str, options: WriteOptions | None def encrypt_package(package_path: str | Path, destination_path: str | Path, password: str) -> None: """Wraps a finished plaintext XLSX/XLSB package in an agile-encrypted (ECMA-376 4.4) CFB - container — the inverse of passing `password=` to `open_workbook`/`open_bytes`. `package_path` + container — the inverse of passing `password=` to `open_workbook`. `package_path` is read twice, so it must already be a complete file (write it with `write_workbook` first). Encryption parameters are fixed at Excel's own defaults; there are no options. """ diff --git a/python/tests/test_reader.py b/python/tests/test_reader.py index 896dbe7f..c5e2741e 100644 --- a/python/tests/test_reader.py +++ b/python/tests/test_reader.py @@ -17,7 +17,6 @@ PasswordIncorrectError, PasswordRequiredError, decode_cell, - open_bytes, open_workbook, ) from excelreader import _native @@ -78,7 +77,7 @@ def test_reads_xls(xls_path): def test_reads_from_bytes(xlsx_path): - with open_bytes(xlsx_path.read_bytes()) as workbook: + with open_workbook(xlsx_path.read_bytes()) as workbook: header = next(workbook.sheets[0].rows()) assert header[0].value == "Coluna1" @@ -432,10 +431,10 @@ def test_open_options_reaches_the_csv_reader(tmp_path): assert [cell.value for cell in next(workbook.sheets[0].rows())] == ["name", "qty", "price"] -def test_open_options_apply_to_open_bytes_too(tmp_path): +def test_open_options_apply_to_bytes_too(tmp_path): data = b"name;qty\nwidget;3\n" - with open_bytes(data, format="csv", options=OpenOptions(csv_delimiter=ord(";"))) as workbook: + with open_workbook(data, format="csv", options=OpenOptions(csv_delimiter=ord(";"))) as workbook: assert [cell.value for cell in next(workbook.sheets[0].rows())] == ["name", "qty"] diff --git a/python/tests/test_sheets.py b/python/tests/test_sheets.py index aa487f06..0ab0c98b 100644 --- a/python/tests/test_sheets.py +++ b/python/tests/test_sheets.py @@ -3,7 +3,7 @@ from concurrent.futures import ThreadPoolExecutor import pytest -from excelreader import ExcelReaderError, Sheet, SheetVisibility, Workbook, open_bytes, open_writer_to_memory +from excelreader import ExcelReaderError, Sheet, SheetVisibility, Workbook, open_workbook, open_writer_to_memory def _two_sheets() -> bytes: @@ -35,13 +35,13 @@ def _many_sheets() -> bytes: def test_a_sheet_reads_its_own_rows(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: assert _first_column(workbook.sheets[1]) == ["b1", "b2", "b3"] assert _first_column(workbook.sheets[0]) == ["a1", "a2"] def test_sheets_is_a_sequence_of_sheets(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: sheets = workbook.sheets assert len(sheets) == 2 @@ -52,19 +52,19 @@ def test_sheets_is_a_sequence_of_sheets(): def test_sheets_looks_a_sheet_up_by_name_without_regard_to_case(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: assert workbook.sheets["Second"].index == 1 assert workbook.sheets["sEcOnD"].index == 1 @pytest.mark.parametrize(("key", "error"), [(5, IndexError), (-3, IndexError), ("nope", KeyError), (1.0, TypeError)]) def test_sheets_rejects_a_key_that_names_no_sheet(key, error): - with open_bytes(_two_sheets()) as workbook, pytest.raises(error): + with open_workbook(_two_sheets()) as workbook, pytest.raises(error): workbook.sheets[key] def test_two_cursors_on_one_sheet_each_return_every_row(): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: sheet = workbook.sheets[1] one = sheet.rows() two = sheet.rows() @@ -76,7 +76,7 @@ def test_two_cursors_on_one_sheet_each_return_every_row(): def test_an_abandoned_rows_generator_closes_its_cursor(monkeypatch): - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: sheet = workbook.sheets[1] closed = [] real_close = workbook._lib.xl_rows_close @@ -94,7 +94,7 @@ def test_an_abandoned_rows_generator_closes_its_cursor(monkeypatch): def test_a_sheet_of_a_closed_workbook_raises(): - workbook = open_bytes(_two_sheets()) + workbook = open_workbook(_two_sheets()) sheet = workbook.sheets[0] workbook.close() @@ -105,7 +105,7 @@ def test_a_sheet_of_a_closed_workbook_raises(): def test_rows_opened_before_close_read_to_the_end_after_it(): - workbook = open_bytes(_two_sheets()) + workbook = open_workbook(_two_sheets()) rows = workbook.sheets[1].rows() assert next(rows)[0].value == "b1" @@ -117,7 +117,7 @@ def test_rows_opened_before_close_read_to_the_end_after_it(): def test_read_all_and_parse_typed_read_the_named_sheet(): from excelreader import ColumnSpec, ColumnType - with open_bytes(_two_sheets()) as workbook: + with open_workbook(_two_sheets()) as workbook: second = workbook.sheets[1] schema = [ColumnSpec(ColumnType.STRING, index=0, nullable=True)] @@ -129,7 +129,7 @@ def test_read_all_and_parse_typed_read_the_named_sheet(): def test_every_sheet_read_on_its_own_thread_matches_a_sequential_read(): - with open_bytes(_many_sheets()) as workbook: + with open_workbook(_many_sheets()) as workbook: sheets = list(workbook.sheets) sequential = [_first_column(sheet) for sheet in sheets] assert sequential[3][0] == "s3-r0" @@ -146,7 +146,7 @@ def test_typed_parses_of_every_sheet_run_on_separate_threads(): from excelreader import ColumnSpec, ColumnType schema = [ColumnSpec(ColumnType.I64, index=1)] - with open_bytes(_many_sheets()) as workbook, ThreadPoolExecutor(max_workers=_PARALLEL_SHEETS) as pool: + with open_workbook(_many_sheets()) as workbook, ThreadPoolExecutor(max_workers=_PARALLEL_SHEETS) as pool: tables = list(pool.map(lambda sheet: sheet.parse_typed(schema, header_row=0), workbook.sheets)) for sheet, table in enumerate(tables): @@ -159,7 +159,7 @@ def test_a_record_batch_reader_outlives_its_workbook(): from excelreader import ColumnSpec, ColumnType schema = [ColumnSpec(ColumnType.I64, index=1)] - with open_bytes(_many_sheets()) as workbook: + with open_workbook(_many_sheets()) as workbook: reader = workbook.sheets[2].to_record_batch_reader(schema, header_row=0, batch_size=100) table = reader.read_all() @@ -171,7 +171,7 @@ def test_a_record_batch_reader_outlives_its_workbook(): def test_sheets_compare_by_workbook_and_index(): data = _two_sheets() - with open_bytes(data) as workbook, open_bytes(data) as other: + with open_workbook(data) as workbook, open_workbook(data) as other: sheets = workbook.sheets assert sheets.index(sheets[1]) == 1 assert sheets[0] in sheets @@ -187,7 +187,7 @@ def test_a_sheet_with_no_rows_reads_empty(): writer.end_sheet() data = writer.bytes() - with open_bytes(data) as workbook: + with open_workbook(data) as workbook: sheet = workbook.sheets[0] assert sheet.read_all() == [] columnar = sheet.read_all_columnar() @@ -219,7 +219,7 @@ def _handle(self, value): def test_closing_from_two_threads_at_once_raises_nowhere(): - source = open_bytes(_two_sheets()) + source = open_workbook(_two_sheets()) workbook = _SlowCheckWorkbook(source._handle) source._handle = None errors: list[BaseException] = [] diff --git a/python/tests/test_sources.py b/python/tests/test_sources.py index a8353732..288a7fbe 100644 --- a/python/tests/test_sources.py +++ b/python/tests/test_sources.py @@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import pytest -from excelreader import ExcelReaderError, OpenOptions, open_bytes, open_source, open_stream, open_writer_to_memory +from excelreader import ExcelReaderError, OpenOptions, open_workbook, open_writer_to_memory from excelreader import sources as sources_module SHEETS = 4 @@ -59,10 +59,10 @@ def read(self, size: int) -> bytes: def test_every_sheet_of_a_source_reads_in_parallel_like_memory(): data = _workbook_bytes() - with open_bytes(data) as reference: + with open_workbook(data) as reference: expected = [_first_column(sheet) for sheet in reference.sheets] - with open_source(BytesSource(data)) as workbook, ThreadPoolExecutor(max_workers=SHEETS * 2) as pool: + with open_workbook(BytesSource(data)) as workbook, ThreadPoolExecutor(max_workers=SHEETS * 2) as pool: sheets = list(workbook.sheets) parallel = list(pool.map(_first_column, sheets + sheets)) @@ -74,7 +74,7 @@ def test_the_source_is_dropped_after_the_workbook_closes(): source = BytesSource(_workbook_bytes()) finalized = weakref.finalize(source, lambda: None) before = set(sources_module._live) - workbook = open_source(source, "xlsx") + workbook = open_workbook(source, "xlsx") assert len(sources_module._live) == len(before) + 1 del source _first_column(workbook.sheets[0]) @@ -86,32 +86,32 @@ def test_the_source_is_dropped_after_the_workbook_closes(): def test_a_source_error_reaches_the_caller_with_its_message(): with pytest.raises(ExcelReaderError, match="disk on fire"): - open_source(FailingSource(), "xlsx") + open_workbook(FailingSource(), "xlsx") def test_the_default_cache_fetches_a_small_workbook_once(): data = _workbook_bytes() cached = BytesSource(data) - with open_source(cached, "xlsx") as workbook: + with open_workbook(cached, "xlsx") as workbook: _first_column(workbook.sheets[0]) assert cached.reads == 1 uncached = BytesSource(data) - with open_source(uncached, "xlsx", options=OpenOptions(source_block_size=-1)) as workbook: + with open_workbook(uncached, "xlsx", options=OpenOptions(source_block_size=-1)) as workbook: _first_column(workbook.sheets[0]) assert uncached.reads > 1 def test_one_source_object_opened_twice_at_once_serves_both(): source = BytesSource(_workbook_bytes()) - first = open_source(source, "xlsx") - second = open_source(source, "xlsx") + first = open_workbook(source, "xlsx") + second = open_workbook(source, "xlsx") first.close() assert _first_column(second.sheets[2])[0] == "s2-r0" second.close() -def test_open_stream_reads_a_csv_from_a_pipe(): +def test_a_stream_reads_a_csv_from_a_pipe(): read_fd, write_fd = os.pipe() def produce() -> None: @@ -122,7 +122,7 @@ def produce() -> None: producer = threading.Thread(target=produce) producer.start() - with os.fdopen(read_fd, "rb") as pipe, open_stream(pipe, "csv") as workbook: + with os.fdopen(read_fd, "rb") as pipe, open_workbook(pipe, "csv") as workbook: names = _first_column(workbook.sheets[0]) producer.join() assert names[0] == "name" @@ -132,7 +132,7 @@ def produce() -> None: def test_a_csv_stream_serves_a_schema_inference_and_then_every_row(): data = b"name,qty\n" + b"".join(f"row{i},{i}\n".encode() for i in range(5000)) - with open_stream(OnlyRead(data), "csv") as workbook: + with open_workbook(OnlyRead(data), "csv") as workbook: sheet = workbook.sheets[0] schema = sheet.infer_schema(parse_text=True) names = _first_column(sheet) @@ -141,59 +141,94 @@ def test_a_csv_stream_serves_a_schema_inference_and_then_every_row(): assert names[-1] == "row4999" -def test_open_stream_reads_an_xlsx_from_an_object_with_only_read(): - with open_stream(OnlyRead(_workbook_bytes())) as workbook: +def test_a_stream_reads_an_xlsx_from_an_object_with_only_read(): + with open_workbook(OnlyRead(_workbook_bytes())) as workbook: rows = _first_column(workbook.sheets[3]) assert rows[0] == "s3-r0" assert len(rows) == ROWS -def test_open_stream_rejects_an_object_that_cannot_be_read(): - with pytest.raises(TypeError): - open_stream(object()) +def test_open_workbook_rejects_an_object_it_cannot_read(): + with pytest.raises(TypeError, match="object"): + open_workbook(object()) + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda data, path: str(path), id="str"), + pytest.param(lambda data, path: path, id="path"), + pytest.param(lambda data, path: data, id="bytes"), + pytest.param(lambda data, path: bytearray(data), id="bytearray"), + pytest.param(lambda data, path: memoryview(data), id="memoryview"), + pytest.param(lambda data, path: io.BytesIO(data), id="bytesio"), + pytest.param(lambda data, path: BytesSource(data), id="source"), + pytest.param(lambda data, path: OnlyRead(data), id="stream"), + pytest.param(lambda data, path: path.open("rb"), id="file"), + ], +) +def test_open_workbook_reads_the_same_rows_from_every_kind_of_input(tmp_path, wrap): + data = _workbook_bytes() + path = tmp_path / "book.xlsx" + path.write_bytes(data) + source = wrap(data, path) + try: + with open_workbook(source) as workbook: + rows = [_first_column(sheet) for sheet in workbook.sheets] + finally: + if hasattr(source, "close"): + source.close() + assert rows == [[f"s{sheet}-r{row}" for row in range(ROWS)] for sheet in range(SHEETS)] + + +def test_a_bytesio_is_read_from_its_start_whatever_its_position(): + buffer = io.BytesIO(_workbook_bytes()) + buffer.seek(0, io.SEEK_END) + with open_workbook(buffer) as workbook: + assert _first_column(workbook.sheets[0])[0] == "s0-r0" def test_a_failed_open_leaves_nothing_registered(): before = set(sources_module._live) source = BytesSource(_workbook_bytes()) with pytest.raises(ValueError): - open_source(source, "nope") + open_workbook(source, "nope") with pytest.raises(ValueError): - open_source(source, "xlsx", password="\ud800") + open_workbook(source, "xlsx", password="\ud800") negative = BytesSource(b"") negative.size = -1 with pytest.raises(ValueError): - open_source(negative, "xlsx") + open_workbook(negative, "xlsx") with pytest.raises(ValueError): - open_stream(io.BytesIO(), "nope") + open_workbook(OnlyRead(b""), "nope") assert set(sources_module._live) == before -def test_open_stream_fails_on_a_text_stream_and_releases_it(): +def test_a_stream_fails_on_a_text_stream_and_releases_it(): before = set(sources_module._live) - with pytest.raises(ExcelReaderError), open_stream(io.StringIO("a,b\n1,2\n"), "csv") as workbook: + with pytest.raises(ExcelReaderError), open_workbook(io.StringIO("a,b\n1,2\n"), "csv") as workbook: _first_column(workbook.sheets[0]) assert set(sources_module._live) == before -def test_open_stream_fails_when_read_returns_more_than_asked(): +def test_a_stream_fails_when_read_returns_more_than_asked(): class Greedy: def read(self, size: int) -> bytes: return b"x" * (size + 1) before = set(sources_module._live) - with pytest.raises(ExcelReaderError, match="more bytes than requested"), open_stream(Greedy(), "csv") as workbook: + with pytest.raises(ExcelReaderError, match="more bytes than requested"), open_workbook(Greedy(), "csv") as workbook: _first_column(workbook.sheets[0]) assert set(sources_module._live) == before -def test_open_stream_fails_when_read_raises_keyboard_interrupt_and_releases_it(): +def test_a_stream_fails_when_read_raises_keyboard_interrupt_and_releases_it(): class Interrupted: def read(self, size: int) -> bytes: raise KeyboardInterrupt before = set(sources_module._live) - with pytest.raises(ExcelReaderError), open_stream(Interrupted(), "csv") as workbook: + with pytest.raises(ExcelReaderError), open_workbook(Interrupted(), "csv") as workbook: _first_column(workbook.sheets[0]) assert set(sources_module._live) == before @@ -206,4 +241,4 @@ def read_at(self, offset: int, buffer: memoryview): return None with pytest.raises(ExcelReaderError): - open_source(Bad(), "xlsx") + open_workbook(Bad(), "xlsx") diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index de2da3dc..6b518dad 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -7,7 +7,6 @@ WriteOptions, encrypt_package, encrypt_package_bytes, - open_bytes, open_workbook, write_arrow, write_arrow_to_bytes, @@ -228,7 +227,7 @@ def test_encrypt_package_rejects_an_empty_password(source_csv, tmp_path): def _parse_bytes(payload, format, password=None): - with open_bytes(payload, format=format, password=password) as workbook: + with open_workbook(payload, format=format, password=password) as workbook: return workbook.sheets[0].parse_typed(_SCHEMA) @@ -260,7 +259,7 @@ def test_encrypt_package_bytes_round_trips_without_touching_disk(source_csv): assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] with pytest.raises(PasswordIncorrectError): - open_bytes(encrypted, format="xlsx", password="wrong") + open_workbook(encrypted, format="xlsx", password="wrong") def test_encrypt_package_bytes_rejects_bytes_that_are_not_a_package(): diff --git a/python/tests/test_writer_handle.py b/python/tests/test_writer_handle.py index 7b3b6042..b80929ed 100644 --- a/python/tests/test_writer_handle.py +++ b/python/tests/test_writer_handle.py @@ -169,7 +169,7 @@ def read(path): def test_in_memory_writer_round_trips(): - from excelreader import open_bytes, open_writer_to_memory + from excelreader import open_workbook, open_writer_to_memory with open_writer_to_memory("xlsx") as writer: writer.start_sheet("Data") @@ -181,7 +181,7 @@ def test_in_memory_writer_round_trips(): assert isinstance(payload, bytes) assert len(payload) > 0 - with open_bytes(payload, format="xlsx") as workbook: + with open_workbook(payload, format="xlsx") as workbook: rows = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] assert rows[0] == ["name", "qty"] @@ -201,7 +201,7 @@ def test_bytes_on_a_file_writer_raises(tmp_path): def test_write_workbook_to_bytes_matches_write_workbook(tmp_path, xlsx_path): - from excelreader import open_bytes, write_workbook, write_workbook_to_bytes + from excelreader import open_workbook, write_workbook, write_workbook_to_bytes with open_workbook(xlsx_path) as workbook: schema = workbook.sheets[0].infer_schema() @@ -215,7 +215,7 @@ def test_write_workbook_to_bytes_matches_write_workbook(tmp_path, xlsx_path): with open_workbook(path) as workbook: from_file = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] - with open_bytes(payload, format="xlsx") as workbook: + with open_workbook(payload, format="xlsx") as workbook: from_memory = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] assert from_file == from_memory From 7b2a46d601e12b980421a012695f0604e2a9d006 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 16:00:43 -0300 Subject: [PATCH 05/21] feat(python)!: write to a path or a binary stream through one write function each write_workbook(), write_arrow(), write_pandas() and write_polars() take the destination first: a path, or any binary object with write(), such as io.BytesIO or an open file. The four *_to_bytes() twins are removed. A destination with no path needs format=, and the frame writers take format and options as keywords instead of forwarding **kwargs. BREAKING CHANGE: write_*_to_bytes(x, format=f) becomes write_*(buffer, x, format=f) on an io.BytesIO. --- docs/guide/migrating-to-v7.md | 9 +++ python/README.md | 32 ++++---- python/src/excelreader/__init__.py | 8 -- python/src/excelreader/writer.py | 113 +++++++++++++++-------------- python/tests/test_writer.py | 55 +++++++++++--- python/tests/test_writer_handle.py | 10 ++- 6 files changed, 139 insertions(+), 88 deletions(-) diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md index 7e453286..fd87c509 100644 --- a/docs/guide/migrating-to-v7.md +++ b/docs/guide/migrating-to-v7.md @@ -169,3 +169,12 @@ The C ABI is unchanged: no signature moved, and `XL_ABI_VERSION` stays 6. Behavi |---|---| | `open_bytes(data, "xlsx", password=p)` | `open_workbook(data, "xlsx", password=p)` | | `open_bytes(buffer.getvalue())` | `open_workbook(buffer)` | + +### Python: one way to write a workbook + +`write_workbook_to_bytes()`, `write_arrow_to_bytes()`, `write_pandas_to_bytes()` and `write_polars_to_bytes()` are removed. `write_workbook()`, `write_arrow()`, `write_pandas()` and `write_polars()` take the destination first, a path or a binary object with `write()`, such as `io.BytesIO`. A destination without a path needs `format`. The frame writers now take `format` and `options` as keywords instead of passing `**kwargs` through. + +| v6 | v7 | +|---|---| +| `payload = write_workbook_to_bytes(table, types, format="xlsx")` | `buffer = io.BytesIO()`
`write_workbook(buffer, table, types, format="xlsx")`
`payload = buffer.getvalue()` | +| `write_pandas_to_bytes(df, format="xlsx")` | `write_pandas(buffer, df, format="xlsx")` | diff --git a/python/README.md b/python/README.md index 476d9f2d..4a2406d8 100644 --- a/python/README.md +++ b/python/README.md @@ -254,8 +254,20 @@ arrays. `format` is inferred from the path's extension (one of xlsx/xlsb/xls/csv write_workbook("report.dat", table, types, format="csv") ``` +The destination can also be a binary object with `write()`, such as `io.BytesIO` or an open file. The +workbook is built in memory and written in one `write()` call, and with no extension to infer it from, +`format` is required: + +```python +import io + +buffer = io.BytesIO() +write_workbook(buffer, table, types, format="xlsx") +payload = buffer.getvalue() +``` + `write_pandas()` and `write_polars()` build the table from a DataFrame instead (both go through -`write_arrow()`, so `pyarrow` must be installed): +`write_arrow()`, so `pyarrow` must be installed). They take the same destinations and keywords: ```python from excelreader import write_pandas, write_polars @@ -315,16 +327,6 @@ with open_writer_to_memory("xlsx") as writer: payload = writer.bytes() ``` -`write_workbook_to_bytes()` is the same idea for the columnar `write_workbook()` path, and -`write_arrow_to_bytes()`, `write_pandas_to_bytes()` and `write_polars_to_bytes()` for the frame -writers. With no path to infer it from, each takes an explicit `format`: - -```python -from excelreader import write_pandas_to_bytes - -payload = write_pandas_to_bytes(df, format="xlsx") -``` - ### Arrow With `pyarrow` installed, `to_arrow()` runs the same read and hands the buffers to pyarrow zero-copy @@ -510,9 +512,13 @@ packages can be encrypted, matching what `open_workbook` can decrypt. to disk: ```python -from excelreader import encrypt_package_bytes, write_workbook_to_bytes +import io + +from excelreader import encrypt_package_bytes, write_workbook -secret = encrypt_package_bytes(write_workbook_to_bytes(table, types, format="xlsx"), "hunter2") +buffer = io.BytesIO() +write_workbook(buffer, table, types, format="xlsx") +secret = encrypt_package_bytes(buffer.getvalue(), "hunter2") ``` ## Benchmarks diff --git a/python/src/excelreader/__init__.py b/python/src/excelreader/__init__.py index 0d54a7e3..f8ecc677 100644 --- a/python/src/excelreader/__init__.py +++ b/python/src/excelreader/__init__.py @@ -22,13 +22,9 @@ encrypt_package, encrypt_package_bytes, write_arrow, - write_arrow_to_bytes, write_pandas, - write_pandas_to_bytes, write_polars, - write_polars_to_bytes, write_workbook, - write_workbook_to_bytes, ) from excelreader.writer_handle import SheetWriter, open_writer, open_writer_to_memory @@ -59,11 +55,7 @@ "open_writer", "open_writer_to_memory", "write_arrow", - "write_arrow_to_bytes", "write_pandas", - "write_pandas_to_bytes", "write_polars", - "write_polars_to_bytes", "write_workbook", - "write_workbook_to_bytes", ] diff --git a/python/src/excelreader/writer.py b/python/src/excelreader/writer.py index abc4196e..68f480e9 100644 --- a/python/src/excelreader/writer.py +++ b/python/src/excelreader/writer.py @@ -1,4 +1,4 @@ -"""Writing a columnar table to a workbook file. Read cursors live in reader.py; this module has none. +"""Writing a columnar table to a workbook file or a binary stream. Read cursors live in reader.py; this module has none. Everything here funnels into the single `xl_write_typed` export. Each column's buffer is copied once into a ctypes block that this module owns for the duration of the call: `xl_write_typed` borrows @@ -11,9 +11,10 @@ import ctypes import datetime +import os from array import array from pathlib import Path -from typing import Any +from typing import IO, Any, Union from excelreader import _native from excelreader.reader import _check @@ -121,31 +122,53 @@ def _marshal( return specs, native_table, raw_options, keepalive +Destination = Union[str, os.PathLike, IO[bytes]] + + def write_workbook( - path: str | Path, + destination: Destination, table: TypedTable, types: list[ColumnType], *, format: str | None = None, options: WriteOptions | None = None, ) -> None: - """Writes `table` to `path` as a single sheet. + """Writes `table` as a single sheet to `destination`: a path, or a binary object with `write()` + such as `io.BytesIO` or an open file. `table` is what `Sheet.parse_typed()` returns, so reading a sheet and writing it back out is a round-trip through the same buffers. `types` gives each column's `ColumnType` and is required positionally: a `TypedTable` carries raw buffers (an `array`/`StringColumn`/NumPy array), and nothing about a raw buffer's element size alone distinguishes, say, I64 from TIME — both are 8-byte-per-row arrays. The type tag has to travel with the call, not be guessed from the data. - `format` is one of xlsx/xlsb/xls/csv; None infers it from the path's extension. + + `format` is one of xlsx/xlsb/xls/csv. For a path, None infers it from the extension; an object + with `write()` has no extension, so it needs `format`. The whole workbook is built in memory and + handed to `write()` in one call, at the object's current position; the object is not closed. `table.names` becomes the header row. `options` sets the sheet name and CSV dialect; see `WriteOptions`. """ - resolved = Path(path) - format_id = _resolve_write_format(format, resolved) - specs, native_table, raw_options, keepalive = _marshal(table, types, options) + if isinstance(destination, (str, os.PathLike)): + _write_file(Path(destination), table, types, format, options) + return + if not hasattr(destination, "write"): + raise TypeError( + f"destination must be a path or an object with write(); got {type(destination).__name__}" + ) + if format is None: + raise ValueError( + f"format= is required when writing to an object with write(); one of {sorted(_FORMAT_NAMES)}" + ) + destination.write(_write_memory(table, types, format, options)) + - encoded = str(resolved).encode("utf-8") +def _write_file( + path: Path, table: TypedTable, types: list[ColumnType], format: str | None, options: WriteOptions | None +) -> None: + format_id = _resolve_write_format(format, path) + specs, native_table, raw_options, keepalive = _marshal(table, types, options) + encoded = str(path).encode("utf-8") lib = _native.load_library() _check( lib.xl_write_typed( @@ -155,32 +178,19 @@ def write_workbook( del keepalive -def write_workbook_to_bytes( - table: TypedTable, - types: list[ColumnType], - *, - format: str, - options: WriteOptions | None = None, +def _write_memory( + table: TypedTable, types: list[ColumnType], format: str, options: WriteOptions | None ) -> bytes: - """The in-memory twin of `write_workbook()`. - - `format` is required and is one of xlsx/xlsb/xls/csv — unlike `write_workbook`, there is no path - to infer it from. - """ - if format not in _native.WRITE_FORMATS: - raise ValueError(f"format must be one of {sorted(_native.WRITE_FORMATS)}; got {format!r}") - + try: + format_id = _FORMAT_NAMES[format.lower()] + except KeyError: + raise ValueError(f"unknown format {format!r}; expected one of {sorted(_FORMAT_NAMES)}") from None specs, native_table, raw_options, keepalive = _marshal(table, types, options) - buffer = _native.NativeBuffer() lib = _native.load_library() _check( lib.xl_write_typed_to_memory( - _native.WRITE_FORMATS[format], - specs, - ctypes.byref(native_table), - ctypes.byref(raw_options), - ctypes.byref(buffer), + format_id, specs, ctypes.byref(native_table), ctypes.byref(raw_options), ctypes.byref(buffer) ) ) try: @@ -320,36 +330,29 @@ def _arrow_from_pandas(df: Any) -> Any: return pyarrow.Table.from_pandas(df, preserve_index=False) -def write_arrow(path: str | Path, batch: Any, **kwargs: Any) -> None: - """Writes a `pyarrow.RecordBatch` (or `Table`) to `path`. Requires pyarrow.""" - table, types = _table_from_arrow(batch) - write_workbook(path, table, types, **kwargs) - - -def write_arrow_to_bytes(batch: Any, *, format: str, options: WriteOptions | None = None) -> bytes: - """The in-memory twin of `write_arrow()`. `format` is required, as in `write_workbook_to_bytes()`.""" +def write_arrow( + destination: Destination, batch: Any, *, format: str | None = None, options: WriteOptions | None = None +) -> None: + """Writes a `pyarrow.RecordBatch` (or `Table`) to `destination`, as `write_workbook()` does. + Requires pyarrow.""" table, types = _table_from_arrow(batch) - return write_workbook_to_bytes(table, types, format=format, options=options) - - -def write_pandas(path: str | Path, df: Any, **kwargs: Any) -> None: - """Writes a `pandas.DataFrame` to `path`. Requires pyarrow and pandas.""" - write_arrow(path, _arrow_from_pandas(df), **kwargs) - + write_workbook(destination, table, types, format=format, options=options) -def write_pandas_to_bytes(df: Any, *, format: str, options: WriteOptions | None = None) -> bytes: - """The in-memory twin of `write_pandas()`. Requires pyarrow and pandas.""" - return write_arrow_to_bytes(_arrow_from_pandas(df), format=format, options=options) - -def write_polars(path: str | Path, df: Any, **kwargs: Any) -> None: - """Writes a `polars.DataFrame` to `path`. Requires pyarrow and polars.""" - write_arrow(path, df.to_arrow(), **kwargs) +def write_pandas( + destination: Destination, df: Any, *, format: str | None = None, options: WriteOptions | None = None +) -> None: + """Writes a `pandas.DataFrame` to `destination`, as `write_workbook()` does. Requires pyarrow and + pandas.""" + write_arrow(destination, _arrow_from_pandas(df), format=format, options=options) -def write_polars_to_bytes(df: Any, *, format: str, options: WriteOptions | None = None) -> bytes: - """The in-memory twin of `write_polars()`. Requires pyarrow and polars.""" - return write_arrow_to_bytes(df.to_arrow(), format=format, options=options) +def write_polars( + destination: Destination, df: Any, *, format: str | None = None, options: WriteOptions | None = None +) -> None: + """Writes a `polars.DataFrame` to `destination`, as `write_workbook()` does. Requires pyarrow and + polars.""" + write_arrow(destination, df.to_arrow(), format=format, options=options) def encrypt_package(package_path: str | Path, destination_path: str | Path, password: str) -> None: @@ -373,7 +376,7 @@ def encrypt_package(package_path: str | Path, destination_path: str | Path, pass def encrypt_package_bytes(package: bytes, password: str) -> bytes: """The in-memory twin of `encrypt_package()`: takes a finished XLSX/XLSB package's bytes (from - `write_workbook_to_bytes()` or a `*_to_bytes` writer) and returns the encrypted workbook's bytes, + `write_workbook()` into an `io.BytesIO`, say) and returns the encrypted workbook's bytes, so the plaintext never has to be written to disk. """ password_encoded = password.encode("utf-8") diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index 6b518dad..67526817 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -1,3 +1,5 @@ +import io + import pytest from excelreader import ( ColumnSpec, @@ -9,13 +11,9 @@ encrypt_package_bytes, open_workbook, write_arrow, - write_arrow_to_bytes, write_pandas, - write_pandas_to_bytes, write_polars, - write_polars_to_bytes, write_workbook, - write_workbook_to_bytes, ) _SCHEMA = [ @@ -231,11 +229,17 @@ def _parse_bytes(payload, format, password=None): return workbook.sheets[0].parse_typed(_SCHEMA) +def _to_bytes(table): + buffer = io.BytesIO() + write_workbook(buffer, table, _TYPES, format="xlsx") + return buffer.getvalue() + + @pytest.mark.parametrize( ("writer", "frame_module"), - [(write_arrow_to_bytes, None), (write_pandas_to_bytes, "pandas"), (write_polars_to_bytes, "polars")], + [(write_arrow, None), (write_pandas, "pandas"), (write_polars, "polars")], ) -def test_frame_writers_to_bytes_round_trip(writer, frame_module): +def test_frame_writers_round_trip_through_a_bytesio(writer, frame_module): pa = pytest.importorskip("pyarrow") data = {"name": ["widget", "gadget"], "qty": [3, 7]} if frame_module is None: @@ -243,7 +247,9 @@ def test_frame_writers_to_bytes_round_trip(writer, frame_module): else: frame = pytest.importorskip(frame_module).DataFrame(data) - result = _parse_bytes(writer(frame, format="xlsx", options=WriteOptions(sheet_name="Vendas")), "xlsx") + buffer = io.BytesIO() + writer(buffer, frame, format="xlsx", options=WriteOptions(sheet_name="Vendas")) + result = _parse_bytes(buffer.getvalue(), "xlsx") assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] @@ -253,7 +259,7 @@ def test_encrypt_package_bytes_round_trips_without_touching_disk(source_csv): with open_workbook(source_csv) as workbook: table = workbook.sheets[0].parse_typed(_SCHEMA) - encrypted = encrypt_package_bytes(write_workbook_to_bytes(table, _TYPES, format="xlsx"), "hunter2") + encrypted = encrypt_package_bytes(_to_bytes(table), "hunter2") result = _parse_bytes(encrypted, "xlsx", password="hunter2") assert list(result.columns[0]) == ["widget", "gadget"] @@ -272,4 +278,35 @@ def test_encrypt_package_bytes_rejects_an_empty_password(source_csv): table = workbook.sheets[0].parse_typed(_SCHEMA) with pytest.raises(ExcelReaderError): - encrypt_package_bytes(write_workbook_to_bytes(table, _TYPES, format="xlsx"), "") + encrypt_package_bytes(_to_bytes(table), "") + + +def test_write_workbook_writes_into_an_open_binary_file(tmp_path, source_csv): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse_typed(_SCHEMA) + path = tmp_path / "out.bin" + + with path.open("wb") as file: + write_workbook(file, table, _TYPES, format="xlsb") + + result = _parse_bytes(path.read_bytes(), "xlsb") + assert list(result.columns[0]) == ["widget", "gadget"] + assert list(result.columns[1]) == [3, 7] + + +def test_write_workbook_to_a_stream_needs_a_format(source_csv): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse_typed(_SCHEMA) + + with pytest.raises(ValueError, match="format"): + write_workbook(io.BytesIO(), table, _TYPES) + with pytest.raises(ValueError, match="unknown format"): + write_workbook(io.BytesIO(), table, _TYPES, format="parquet") + + +def test_write_workbook_rejects_a_destination_it_cannot_write(source_csv): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse_typed(_SCHEMA) + + with pytest.raises(TypeError, match="destination"): + write_workbook(42, table, _TYPES, format="xlsx") diff --git a/python/tests/test_writer_handle.py b/python/tests/test_writer_handle.py index b80929ed..f242143c 100644 --- a/python/tests/test_writer_handle.py +++ b/python/tests/test_writer_handle.py @@ -200,8 +200,10 @@ def test_bytes_on_a_file_writer_raises(tmp_path): writer.bytes() -def test_write_workbook_to_bytes_matches_write_workbook(tmp_path, xlsx_path): - from excelreader import open_workbook, write_workbook, write_workbook_to_bytes +def test_write_workbook_to_a_bytesio_matches_write_workbook_to_a_file(tmp_path, xlsx_path): + import io + + from excelreader import open_workbook, write_workbook with open_workbook(xlsx_path) as workbook: schema = workbook.sheets[0].infer_schema() @@ -211,7 +213,9 @@ def test_write_workbook_to_bytes_matches_write_workbook(tmp_path, xlsx_path): path = tmp_path / "from-file.xlsx" write_workbook(path, table, types, format="xlsx") - payload = write_workbook_to_bytes(table, types, format="xlsx") + buffer = io.BytesIO() + write_workbook(buffer, table, types, format="xlsx") + payload = buffer.getvalue() with open_workbook(path) as workbook: from_file = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] From 9b6f5f1b34acfec86782667d04275ac55bfc524a Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 16:06:00 -0300 Subject: [PATCH 06/21] feat(python)!: stream and encrypt to a path or a binary stream through one function each open_writer() takes a path or any binary object with write(); the latter receives the finished workbook in one write() call when the writer closes, even with a sheet left open. open_writer_to_memory() and SheetWriter.bytes() are removed. encrypt_package() takes the package as a path, bytes or io.BytesIO and the destination as a path or an object with write(). Path to path still uses the native file export; any other pair goes through memory. encrypt_package_bytes() is removed. BREAKING CHANGE: open_writer_to_memory(f) becomes open_writer(buffer, f), and encrypt_package_bytes(p, pw) becomes encrypt_package(p, buffer, pw). --- docs/guide/migrating-to-v7.md | 7 ++ python/README.md | 32 +++++---- python/src/excelreader/__init__.py | 5 +- python/src/excelreader/writer.py | 61 ++++++++++------ python/src/excelreader/writer_handle.py | 95 +++++++++++++------------ python/tests/test_sheets.py | 18 +++-- python/tests/test_sources.py | 7 +- python/tests/test_writer.py | 32 ++++++--- python/tests/test_writer_handle.py | 44 ++++++++---- 9 files changed, 184 insertions(+), 117 deletions(-) diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md index fd87c509..9b8df738 100644 --- a/docs/guide/migrating-to-v7.md +++ b/docs/guide/migrating-to-v7.md @@ -178,3 +178,10 @@ The C ABI is unchanged: no signature moved, and `XL_ABI_VERSION` stays 6. Behavi |---|---| | `payload = write_workbook_to_bytes(table, types, format="xlsx")` | `buffer = io.BytesIO()`
`write_workbook(buffer, table, types, format="xlsx")`
`payload = buffer.getvalue()` | | `write_pandas_to_bytes(df, format="xlsx")` | `write_pandas(buffer, df, format="xlsx")` | + +The streaming writer and the encryption helper follow the same pattern. `open_writer_to_memory()` and `SheetWriter.bytes()` are removed: `open_writer()` takes a binary object with `write()` and hands it the finished workbook when the writer closes. `encrypt_package_bytes()` is removed: `encrypt_package()` takes the package as a path, `bytes` or `io.BytesIO`, and the destination as a path or an object with `write()`. + +| v6 | v7 | +|---|---| +| `with open_writer_to_memory("xlsx") as w:`
` ...`
` payload = w.bytes()` | `buffer = io.BytesIO()`
`with open_writer(buffer, "xlsx") as w:`
` ...`
`payload = buffer.getvalue()` | +| `secret = encrypt_package_bytes(payload, password)` | `encrypt_package(payload, out, password)` with `out = io.BytesIO()` | diff --git a/python/README.md b/python/README.md index 4a2406d8..cd3d025a 100644 --- a/python/README.md +++ b/python/README.md @@ -314,17 +314,21 @@ write a typed blank — use the explicit methods: `write_str`, `write_i64`, `wri `write_date`, `write_time`, `write_timestamp`, and `write_null(ColumnType.I64)`. A `None` passed to `write_row` becomes a blank string cell. -To build a workbook without touching the filesystem, use `open_writer_to_memory` and read the result -out with `bytes()`: +To build a workbook without touching the filesystem, open the writer over a binary object with +`write()`, such as `io.BytesIO`. It needs `format`, and receives the finished workbook in one +`write()` call when the writer closes: ```python -from excelreader import open_writer_to_memory +import io + +from excelreader import open_writer -with open_writer_to_memory("xlsx") as writer: +buffer = io.BytesIO() +with open_writer(buffer, "xlsx") as writer: writer.start_sheet("Data") writer.write_row(["name", "qty"]) writer.end_sheet() - payload = writer.bytes() +payload = buffer.getvalue() ``` ### Arrow @@ -504,21 +508,21 @@ write_workbook("plain.xlsx", table, types) encrypt_package("plain.xlsx", "secret.xlsx", "hunter2") ``` -`package_path` is read twice (it is not disposed or removed), so it must already be a finished file. -Encryption parameters are fixed at Excel's own defaults — there are no options — and only XLSX/XLSB -packages can be encrypted, matching what `open_workbook` can decrypt. +The package must already be a finished file; it is read, not removed. Encryption parameters are +fixed at Excel's own defaults — there are no options — and only XLSX/XLSB packages can be encrypted, +matching what `open_workbook` can decrypt. -`encrypt_package_bytes()` does the same over bytes, so the plaintext package never has to be written -to disk: +Either side can stay in memory, so the plaintext package never has to be written to disk: the +package can be `bytes` or an `io.BytesIO`, and the destination any binary object with `write()`: ```python import io -from excelreader import encrypt_package_bytes, write_workbook +from excelreader import encrypt_package, write_workbook -buffer = io.BytesIO() -write_workbook(buffer, table, types, format="xlsx") -secret = encrypt_package_bytes(buffer.getvalue(), "hunter2") +plain, secret = io.BytesIO(), io.BytesIO() +write_workbook(plain, table, types, format="xlsx") +encrypt_package(plain, secret, "hunter2") ``` ## Benchmarks diff --git a/python/src/excelreader/__init__.py b/python/src/excelreader/__init__.py index f8ecc677..826fc891 100644 --- a/python/src/excelreader/__init__.py +++ b/python/src/excelreader/__init__.py @@ -20,13 +20,12 @@ ) from excelreader.writer import ( encrypt_package, - encrypt_package_bytes, write_arrow, write_pandas, write_polars, write_workbook, ) -from excelreader.writer_handle import SheetWriter, open_writer, open_writer_to_memory +from excelreader.writer_handle import SheetWriter, open_writer __all__ = [ "ArrowStream", @@ -50,10 +49,8 @@ "WriteOptions", "decode_cell", "encrypt_package", - "encrypt_package_bytes", "open_workbook", "open_writer", - "open_writer_to_memory", "write_arrow", "write_pandas", "write_polars", diff --git a/python/src/excelreader/writer.py b/python/src/excelreader/writer.py index 68f480e9..d7cd6740 100644 --- a/python/src/excelreader/writer.py +++ b/python/src/excelreader/writer.py @@ -11,6 +11,7 @@ import ctypes import datetime +import io import os from array import array from pathlib import Path @@ -355,39 +356,57 @@ def write_polars( write_arrow(destination, df.to_arrow(), format=format, options=options) -def encrypt_package(package_path: str | Path, destination_path: str | Path, password: str) -> None: +def encrypt_package( + package: Union[str, os.PathLike, bytes, bytearray, memoryview, io.BytesIO], + destination: Destination, + password: str, +) -> None: """Wraps a finished plaintext XLSX/XLSB package in an agile-encrypted (ECMA-376 4.4) CFB - container — the inverse of passing `password=` to `open_workbook`. `package_path` - is read twice, so it must already be a complete file (write it with `write_workbook` first). - Encryption parameters are fixed at Excel's own defaults; there are no options. + container, the inverse of passing `password=` to `open_workbook`. + + `package` is a path to a complete file, or its bytes (`bytes`, `bytearray`, `memoryview`, or an + `io.BytesIO` read whole). `destination` is a path, or a binary object with `write()` that + receives the encrypted workbook in one call and is not closed. Encryption parameters are fixed + at Excel's own defaults; there are no options. """ - package_encoded = str(Path(package_path)).encode("utf-8") - destination_encoded = str(Path(destination_path)).encode("utf-8") password_encoded = password.encode("utf-8") lib = _native.load_library() - _check( - lib.xl_encrypt_package( - package_encoded, len(package_encoded), - destination_encoded, len(destination_encoded), - password_encoded, len(password_encoded), + if isinstance(package, (str, os.PathLike)) and isinstance(destination, (str, os.PathLike)): + package_encoded = str(Path(package)).encode("utf-8") + destination_encoded = str(Path(destination)).encode("utf-8") + _check( + lib.xl_encrypt_package( + package_encoded, len(package_encoded), + destination_encoded, len(destination_encoded), + password_encoded, len(password_encoded), + ) ) - ) + return + if not isinstance(destination, (str, os.PathLike)) and not hasattr(destination, "write"): + raise TypeError( + f"destination must be a path or an object with write(); got {type(destination).__name__}" + ) + if isinstance(package, (str, os.PathLike)): + data = Path(package).read_bytes() + elif isinstance(package, io.BytesIO): + data = package.getvalue() + elif isinstance(package, (bytes, bytearray, memoryview)): + data = bytes(package) + else: + raise TypeError(f"package must be a path or bytes; got {type(package).__name__}") -def encrypt_package_bytes(package: bytes, password: str) -> bytes: - """The in-memory twin of `encrypt_package()`: takes a finished XLSX/XLSB package's bytes (from - `write_workbook()` into an `io.BytesIO`, say) and returns the encrypted workbook's bytes, - so the plaintext never has to be written to disk. - """ - password_encoded = password.encode("utf-8") buffer = _native.NativeBuffer() - lib = _native.load_library() _check( lib.xl_encrypt_package_to_memory( - package, len(package), password_encoded, len(password_encoded), ctypes.byref(buffer) + data, len(data), password_encoded, len(password_encoded), ctypes.byref(buffer) ) ) try: - return ctypes.string_at(buffer.data, buffer.len) + encrypted = ctypes.string_at(buffer.data, buffer.len) finally: lib.xl_free_buffer(ctypes.byref(buffer)) + if isinstance(destination, (str, os.PathLike)): + Path(destination).write_bytes(encrypted) + else: + destination.write(encrypted) diff --git a/python/src/excelreader/writer_handle.py b/python/src/excelreader/writer_handle.py index 589f9dc6..7bfd1c07 100644 --- a/python/src/excelreader/writer_handle.py +++ b/python/src/excelreader/writer_handle.py @@ -9,8 +9,9 @@ import ctypes import datetime +import os from pathlib import Path -from typing import Any, Sequence +from typing import IO, Any, Sequence, Union from excelreader import _native from excelreader.reader import _check @@ -34,19 +35,20 @@ def _is_writable(value: Any) -> bool: class SheetWriter: - """A streaming workbook writer. Use as a context manager; closing finalizes the file. + """A streaming workbook writer. Use as a context manager; closing finalizes the workbook, and for + a destination with `write()` hands it the finished bytes. The required call order is `start_sheet` -> `start_row` -> one write per cell -> `end_row` -> `end_sheet`. Calling out of order raises `ExcelReaderError`, and the writer stays usable: fix the order and continue, or close to discard. """ - __slots__ = ("_handle", "_lib", "_in_memory") + __slots__ = ("_handle", "_lib", "_destination") - def __init__(self, handle: ctypes.c_void_p, in_memory: bool) -> None: + def __init__(self, handle: ctypes.c_void_p, destination: IO[bytes] | None) -> None: self._handle: ctypes.c_void_p | None = handle self._lib = _native.load_library() - self._in_memory = in_memory + self._destination = destination def _require_handle(self) -> ctypes.c_void_p: if self._handle is None: @@ -167,15 +169,9 @@ def _write_inferred(self, value: Any) -> None: else: self.write_str(value) - def bytes(self) -> bytes: - """The workbook written so far, for an in-memory writer. - - Implicitly finishes the workbook's trailing structure, so it is safe whether or not every - sheet and row was explicitly ended. Unlike `close()`, it does not release the writer — call - `close()` afterward either way. Raises `ExcelReaderError` for a file-backed writer. - """ + def _finished_bytes(self, handle: ctypes.c_void_p) -> bytes: buffer = _native.NativeBuffer() - _check(self._lib.xl_write_handle_bytes(self._require_handle(), ctypes.byref(buffer))) + _check(self._lib.xl_write_handle_bytes(handle, ctypes.byref(buffer))) try: if not buffer.data or buffer.len <= 0: return b"" @@ -184,11 +180,20 @@ def bytes(self) -> bytes: self._lib.xl_free_buffer(ctypes.byref(buffer)) def close(self) -> None: - """Finalizes and releases the writer. Safe to call more than once.""" + """Finalizes and releases the writer. Safe to call more than once. + + For a destination with `write()`, the finished workbook is written to it in one call, even + when sheets or rows were left open. + """ if self._handle is None: return handle, self._handle = self._handle, None - _check(self._lib.xl_close_write_handle(handle)) + destination, self._destination = self._destination, None + try: + if destination is not None: + destination.write(self._finished_bytes(handle)) + finally: + _check(self._lib.xl_close_write_handle(handle)) def __enter__(self) -> "SheetWriter": return self @@ -204,47 +209,47 @@ def __del__(self) -> None: def open_writer( - path: str | Path, + destination: Union[str, os.PathLike, IO[bytes]], format: str | None = None, options: WriteOptions | None = None, ) -> SheetWriter: - """Opens a streaming writer for `path`, truncating any existing file. + """Opens a streaming writer for `destination`: a path, truncating any existing file, or a binary + object with `write()` such as `io.BytesIO` or an open file. - `format` is one of xlsx/xlsb/xls/csv; None infers it from the path's extension. + `format` is one of xlsx/xlsb/xls/csv. For a path, None infers it from the extension; an object + with `write()` needs it. Such an object receives the whole workbook in one `write()` call when the + writer closes, and is not closed itself. """ from excelreader.writer import _resolve_write_format - resolved = Path(path) - format_id = _resolve_write_format(format, resolved) raw_options = _native.to_native_write_options(options or WriteOptions()) - - encoded = str(resolved).encode("utf-8") - pointer = ctypes.cast(ctypes.c_char_p(encoded), ctypes.POINTER(ctypes.c_uint8)) handle = ctypes.c_void_p() lib = _native.load_library() - _check( - lib.xl_open_write_handle( - pointer, len(encoded), format_id, ctypes.byref(raw_options), ctypes.byref(handle) - ) - ) - return SheetWriter(handle, in_memory=False) + if isinstance(destination, (str, os.PathLike)): + resolved = Path(destination) + format_id = _resolve_write_format(format, resolved) + encoded = str(resolved).encode("utf-8") + pointer = ctypes.cast(ctypes.c_char_p(encoded), ctypes.POINTER(ctypes.c_uint8)) + _check( + lib.xl_open_write_handle( + pointer, len(encoded), format_id, ctypes.byref(raw_options), ctypes.byref(handle) + ) + ) + return SheetWriter(handle, None) -def open_writer_to_memory(format: str, options: WriteOptions | None = None) -> SheetWriter: - """Opens a streaming writer backed by an in-memory buffer, read out with `SheetWriter.bytes()`. - - `format` is one of xlsx/xlsb/xls/csv — unlike `open_writer`, there is no path to infer it from. - """ - if format not in _native.WRITE_FORMATS: - raise ValueError( - f"format must be one of {sorted(_native.WRITE_FORMATS)}; got {format!r}" + if not hasattr(destination, "write"): + raise TypeError( + f"destination must be a path or an object with write(); got {type(destination).__name__}" ) - raw_options = _native.to_native_write_options(options or WriteOptions()) - handle = ctypes.c_void_p() - lib = _native.load_library() - _check( - lib.xl_open_write_handle_to_memory( - _native.WRITE_FORMATS[format], ctypes.byref(raw_options), ctypes.byref(handle) + if format is None: + raise ValueError( + "format= is required when writing to an object with write(); " + f"one of {sorted(_native.WRITE_FORMATS)}" ) - ) - return SheetWriter(handle, in_memory=True) + try: + format_id = _native.WRITE_FORMATS[format.lower()] + except KeyError: + raise ValueError(f"unknown format {format!r}; expected one of {sorted(_native.WRITE_FORMATS)}") from None + _check(lib.xl_open_write_handle_to_memory(format_id, ctypes.byref(raw_options), ctypes.byref(handle))) + return SheetWriter(handle, destination) diff --git a/python/tests/test_sheets.py b/python/tests/test_sheets.py index 0ab0c98b..4eeb3789 100644 --- a/python/tests/test_sheets.py +++ b/python/tests/test_sheets.py @@ -1,19 +1,21 @@ import gc +import io import threading from concurrent.futures import ThreadPoolExecutor import pytest -from excelreader import ExcelReaderError, Sheet, SheetVisibility, Workbook, open_workbook, open_writer_to_memory +from excelreader import ExcelReaderError, Sheet, SheetVisibility, Workbook, open_workbook, open_writer def _two_sheets() -> bytes: - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: for name, values in (("First", ["a1", "a2"]), ("Second", ["b1", "b2", "b3"])): writer.start_sheet(name) for value in values: writer.write_row([value]) writer.end_sheet() - return writer.bytes() + return buffer.getvalue() def _first_column(sheet: Sheet) -> list[str]: @@ -25,13 +27,14 @@ def _first_column(sheet: Sheet) -> list[str]: def _many_sheets() -> bytes: - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: for sheet in range(_PARALLEL_SHEETS): writer.start_sheet(f"sheet{sheet}") for row in range(_PARALLEL_ROWS): writer.write_row([f"s{sheet}-r{row}", sheet * 1_000_000 + row]) writer.end_sheet() - return writer.bytes() + return buffer.getvalue() def test_a_sheet_reads_its_own_rows(): @@ -182,10 +185,11 @@ def test_sheets_compare_by_workbook_and_index(): def test_a_sheet_with_no_rows_reads_empty(): - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: writer.start_sheet("Empty") writer.end_sheet() - data = writer.bytes() + data = buffer.getvalue() with open_workbook(data) as workbook: sheet = workbook.sheets[0] diff --git a/python/tests/test_sources.py b/python/tests/test_sources.py index 288a7fbe..209c49ef 100644 --- a/python/tests/test_sources.py +++ b/python/tests/test_sources.py @@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import pytest -from excelreader import ExcelReaderError, OpenOptions, open_workbook, open_writer_to_memory +from excelreader import ExcelReaderError, OpenOptions, open_workbook, open_writer from excelreader import sources as sources_module SHEETS = 4 @@ -14,13 +14,14 @@ def _workbook_bytes() -> bytes: - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: for sheet in range(SHEETS): writer.start_sheet(f"sheet{sheet}") for row in range(ROWS): writer.write_row([f"s{sheet}-r{row}"]) writer.end_sheet() - return writer.bytes() + return buffer.getvalue() def _first_column(sheet) -> list[str]: diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index 67526817..d911b7d0 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -8,7 +8,6 @@ PasswordIncorrectError, WriteOptions, encrypt_package, - encrypt_package_bytes, open_workbook, write_arrow, write_pandas, @@ -255,30 +254,47 @@ def test_frame_writers_round_trip_through_a_bytesio(writer, frame_module): assert list(result.columns[1]) == [3, 7] -def test_encrypt_package_bytes_round_trips_without_touching_disk(source_csv): +@pytest.mark.parametrize("package_kind", ["path", "bytes", "bytesio"]) +@pytest.mark.parametrize("destination_kind", ["path", "bytesio"]) +def test_encrypt_package_takes_paths_and_buffers_on_both_sides( + tmp_path, source_csv, package_kind, destination_kind +): with open_workbook(source_csv) as workbook: table = workbook.sheets[0].parse_typed(_SCHEMA) + plain = _to_bytes(table) + plain_path = tmp_path / "plain.xlsx" + plain_path.write_bytes(plain) + package = {"path": plain_path, "bytes": plain, "bytesio": io.BytesIO(plain)}[package_kind] + destination = tmp_path / "secret.xlsx" if destination_kind == "path" else io.BytesIO() - encrypted = encrypt_package_bytes(_to_bytes(table), "hunter2") - result = _parse_bytes(encrypted, "xlsx", password="hunter2") + encrypt_package(package, destination, "hunter2") + encrypted = destination.read_bytes() if destination_kind == "path" else destination.getvalue() + result = _parse_bytes(encrypted, "xlsx", password="hunter2") assert list(result.columns[0]) == ["widget", "gadget"] assert list(result.columns[1]) == [3, 7] with pytest.raises(PasswordIncorrectError): open_workbook(encrypted, format="xlsx", password="wrong") -def test_encrypt_package_bytes_rejects_bytes_that_are_not_a_package(): +def test_encrypt_package_rejects_bytes_that_are_not_a_package(): with pytest.raises(ExcelReaderError): - encrypt_package_bytes(b"not a zip", "hunter2") + encrypt_package(b"not a zip", io.BytesIO(), "hunter2") -def test_encrypt_package_bytes_rejects_an_empty_password(source_csv): +def test_encrypt_package_rejects_an_empty_password_in_memory(source_csv): with open_workbook(source_csv) as workbook: table = workbook.sheets[0].parse_typed(_SCHEMA) with pytest.raises(ExcelReaderError): - encrypt_package_bytes(_to_bytes(table), "") + encrypt_package(_to_bytes(table), io.BytesIO(), "") + + +def test_encrypt_package_rejects_inputs_it_cannot_use(): + with pytest.raises(TypeError, match="package"): + encrypt_package(42, io.BytesIO(), "hunter2") + with pytest.raises(TypeError, match="destination"): + encrypt_package(b"x", 42, "hunter2") def test_write_workbook_writes_into_an_open_binary_file(tmp_path, source_csv): diff --git a/python/tests/test_writer_handle.py b/python/tests/test_writer_handle.py index f242143c..db82d7fe 100644 --- a/python/tests/test_writer_handle.py +++ b/python/tests/test_writer_handle.py @@ -168,36 +168,50 @@ def read(path): assert read(sugar) == read(explicit) -def test_in_memory_writer_round_trips(): - from excelreader import open_workbook, open_writer_to_memory +def test_a_writer_over_a_bytesio_hands_it_the_workbook_on_close(): + import io + + from excelreader import open_workbook - with open_writer_to_memory("xlsx") as writer: + buffer = io.BytesIO() + with open_writer(buffer, "xlsx") as writer: writer.start_sheet("Data") writer.write_row(["name", "qty"]) writer.write_row(["widget", 7]) writer.end_sheet() - payload = writer.bytes() + assert buffer.getvalue() == b"" - assert isinstance(payload, bytes) - assert len(payload) > 0 - - with open_workbook(payload, format="xlsx") as workbook: + assert not buffer.closed + with open_workbook(buffer) as workbook: rows = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] assert rows[0] == ["name", "qty"] assert rows[1][0] == "widget" -def test_bytes_on_a_file_writer_raises(tmp_path): - from excelreader import ExcelReaderError +def test_a_writer_over_a_bytesio_finishes_a_sheet_left_open(): + import io - path = tmp_path / "file.xlsx" - with open_writer(path) as writer: + from excelreader import open_workbook + + buffer = io.BytesIO() + with open_writer(buffer, "xlsb") as writer: writer.start_sheet("Data") writer.write_row(["a"]) - writer.end_sheet() - with pytest.raises(ExcelReaderError): - writer.bytes() + + with open_workbook(buffer) as workbook: + assert [row[0].value for row in workbook.sheets[0].rows()] == ["a"] + + +def test_a_writer_over_a_stream_needs_a_format_and_a_write_method(): + import io + + with pytest.raises(ValueError, match="format"): + open_writer(io.BytesIO()) + with pytest.raises(ValueError, match="unknown format"): + open_writer(io.BytesIO(), "parquet") + with pytest.raises(TypeError, match="destination"): + open_writer(42, "xlsx") def test_write_workbook_to_a_bytesio_matches_write_workbook_to_a_file(tmp_path, xlsx_path): From 699b400ba5a211fa1c70b8715a61dbae33b6ed53 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 16:49:38 -0300 Subject: [PATCH 07/21] feat(python)!: slim the public API to one shape per read and one write function Removed: - Workbook.sheet_count, sheet_names, sheet_name_at (use workbook.sheets) - Sheet.read_all, read_all_columnar, ColumnarSheet, decode_cell - Sheet.to_record_batch (to_arrow now returns the RecordBatch) - Sheet.to_record_batch_reader (use pyarrow.RecordBatchReader.from_stream(sheet.to_arrow_stream(...))) - Sheet.iter_pandas, iter_polars - write_arrow, write_pandas, write_polars (write_workbook takes Arrow data) - SheetWriter.write_str/i64/f64/bool/date/time/timestamp/null, start_row, end_row (write_row infers types; (None, ColumnType.X) is a typed blank) - OpenOptions.csv_max_cell_bytes, csv_intern_strings (max_cell_bytes and intern_strings now reach the CSV reader too) - ArrowStream from __all__ to_pandas gains parallelism, matching to_polars. BREAKING CHANGE: the names above are gone from the Python package; each has a 7.0 replacement listed above. Co-Authored-By: Claude Opus 5.5 --- python/README.md | 154 ++++++------- python/benchmarks/bench_polars_engine.py | 4 +- python/benchmarks/bench_read.py | 52 ++--- python/benchmarks/bench_write.py | 20 +- python/examples/read_workbook.py | 2 +- python/src/excelreader/__init__.py | 18 +- python/src/excelreader/_native.py | 36 +-- python/src/excelreader/reader.py | 268 +++++------------------ python/src/excelreader/types.py | 32 +-- python/src/excelreader/writer.py | 79 +++---- python/src/excelreader/writer_handle.py | 133 ++++------- python/tests/test_reader.py | 177 +++------------ python/tests/test_sheets.py | 17 +- python/tests/test_writer.py | 83 +++++-- python/tests/test_writer_handle.py | 116 +++++----- 15 files changed, 398 insertions(+), 793 deletions(-) diff --git a/python/README.md b/python/README.md index cd3d025a..9a3a9eb9 100644 --- a/python/README.md +++ b/python/README.md @@ -22,7 +22,7 @@ elsewhere, set `EXCELREADER_NATIVE_LIB` to its full path. from excelreader import open_workbook with open_workbook("book.xlsx") as workbook: - print(workbook.sheet_count, workbook.sheets[0].name) + print(len(workbook.sheets), workbook.sheets[0].name) for row in workbook.sheets[0].rows(): for cell in row: print(cell.column, cell.type.name, cell.value) @@ -59,7 +59,7 @@ with excelreader.open_workbook("report.xlsx") as workbook, ThreadPoolExecutor() `rows()` and `iter_parse_typed()` are generators: the native cursor opens at the first `next()`. A generator that has already started keeps working after `workbook.close()`; one that has not started -raises at its first `next()`. A `RecordBatchReader` opens its stream at the call, so it works after +raises at its first `next()`. `to_arrow_stream()` opens its stream at the call, so it works after `close()` even if never started. ### Formats @@ -82,47 +82,28 @@ as_date = cell.as_date(workbook.is_date1904) `as_date()` returns `None` for any cell that isn't `CellType.DATE`. -### Reading everything at once +### Reading a sheet -`rows()` iterates row-by-row; `read_all()` materializes the whole sheet in one call: +A sheet reads in one of these shapes: -```python -all_rows = workbook.sheets[0].read_all() -``` - -This holds every row in memory at once, so prefer `rows()` for very large sheets. - -### Reading everything at once, faster - -`read_all()`/`rows()` build one `Cell`/`str` object per cell, which dominates wall-clock time on a -large sheet. `read_all_columnar()` decodes the same data into parallel flat arrays instead — no -per-cell object construction — and is several times faster on large sheets: - -```python -columnar = workbook.sheets[0].read_all_columnar() -# columnar.row_offsets[i]:row_offsets[i+1] -> cell indices for row i -# columnar.columns[j] / columnar.types[j] -> cell j's column index / CellType -# columnar.value_offsets[j]:[j+1] -> cell j's byte slice into columnar.values -``` +| Method | Returns | Knob | Needs | +|---|---|---|---| +| `rows()` | one list of `Cell` per row, streamed | — | nothing | +| `parse_typed()` | the whole sheet as a `TypedTable` | `parallelism` | nothing | +| `iter_parse_typed()` | one `TypedTable` per batch | `batch_size` | nothing | +| `to_arrow()` | the whole sheet as a `pyarrow.RecordBatch` | `parallelism` | pyarrow | +| `to_arrow_stream()` | an Arrow PyCapsule stream of batches | `batch_size` | nothing | +| `to_pandas()` / `to_polars()` | a DataFrame | `batch_size`, `parallelism` | pandas + pyarrow / polars | -Materialize a single cell on demand instead of decoding every value up front: - -```python -from excelreader import decode_cell - -first_cell = decode_cell(columnar, 0) -``` - -Each array is a stdlib `array.array('i')`, or a NumPy `int32` array if NumPy is installed -(`pip install -e "python[numpy]"`) — NumPy is optional and never required. +`infer_schema()` guesses the schema every typed method takes. ### Typed columns — the fastest path -Everything above hands back cell *text*, which means the library formats every value to a string on -the way out. `parse_typed()` skips that entirely: you give it a schema, and the conversion happens -natively, straight into typed column buffers. On a 65,536 × 14 sheet it is ~8× faster than -`read_all_columnar()`, ~25× faster than `read_all()`, and faster than `polars.read_excel()` — see -[Reading](#reading) below for the measured numbers. +`rows()` hands back cell *text*, which means the library formats every value to a string on the way +out and builds one Python object per cell. `parse_typed()` skips that entirely: you give it a schema, +and the conversion happens natively, straight into typed column buffers. On a 65,536 × 14 sheet it is +~30× faster than `rows()`, and faster than `polars.read_excel()` — see [Reading](#reading) below for +the measured numbers. ```python from excelreader import ColumnSpec, ColumnType @@ -143,6 +124,9 @@ revenue[0] # 14862.69 table.validity[2] # bit-packed nulls, or None when the column has none ``` +Each non-string column is a stdlib `array.array`, or a NumPy array if NumPy is installed +(`pip install -e "python[numpy]"`) — NumPy is optional and never required. + Leave `name` out to resolve a column by position instead: `ColumnSpec(ColumnType.I64, index=3)`. `header_row` defaults to 1 (the first row names the columns); pass `header_row=0` for a sheet with no header, where every spec must resolve by index. @@ -179,12 +163,12 @@ Being a generator, `iter_parse_typed()` opens nothing until the first iteration, #### Reading a large sheet on several threads -`parse_typed()`, `to_arrow()`, `to_record_batch()` and `to_polars()` take `parallelism`: `1` (the +`parse_typed()`, `to_arrow()`, `to_pandas()` and `to_polars()` take `parallelism`: `1` (the default) reads on one thread, `0` uses every core, `n` up to n threads. XLSX, XLSB and CSV are split. XLS, or a sheet under 256 KiB, is read on one thread with the same result. The table is identical either way. Every partition's columns are held until they are merged, so the peak memory is higher -than a sequential read of the same file. `to_polars()` with `parallelism` other than 1 parses the whole -file instead of streaming it in batches. +than a sequential read of the same file. `to_pandas()` and `to_polars()` with `parallelism` other +than 1 parse the whole file instead of streaming it in batches. `xl_parse_typed` on a Ryzen 7 5700X (8 cores, 16 threads), 14 typed columns, file read from memory: @@ -266,14 +250,15 @@ write_workbook(buffer, table, types, format="xlsx") payload = buffer.getvalue() ``` -`write_pandas()` and `write_polars()` build the table from a DataFrame instead (both go through -`write_arrow()`, so `pyarrow` must be installed). They take the same destinations and keywords: +`write_workbook()` also takes Arrow-shaped data in place of a `TypedTable`: a `pyarrow.RecordBatch` +or `Table`, a pandas DataFrame (its index is not written), or any table exposing +`__arrow_c_stream__`, such as a polars DataFrame. Each column's type comes from its Arrow type, so +leave `types` out. This path requires `pyarrow`: ```python -from excelreader import write_pandas, write_polars - -write_pandas("report.xlsx", df) # requires pandas + pyarrow -write_polars("report.xlsx", polars_df) # requires polars + pyarrow +write_workbook("report.xlsx", df) # pandas +write_workbook("report.xlsx", polars_df) # polars +write_workbook("report.xlsx", record_batch) ``` `WriteOptions` sets the sheet name and CSV dialect, mirroring `xl_write_options` — every field @@ -309,10 +294,13 @@ with open_writer("out.xlsx") as writer: writer.end_sheet() ``` -`write_row` picks each cell's type from the Python value. For control over a column's type — or to -write a typed blank — use the explicit methods: `write_str`, `write_i64`, `write_f64`, `write_bool`, -`write_date`, `write_time`, `write_timestamp`, and `write_null(ColumnType.I64)`. A `None` passed to -`write_row` becomes a blank string cell. +`write_row` picks each cell's type from the Python value: `str`, `bool`, `int` (64-bit), `float`, +`datetime.datetime` (an aware one is converted to UTC), `datetime.date` and `datetime.time`. A `None` +becomes a blank string cell; the tuple `(None, ColumnType.I64)` writes a blank cell of that type: + +```python +writer.write_row(["widget", (None, ColumnType.I64), 2.5]) +``` To build a workbook without touching the filesystem, open the writer over a binary object with `write()`, such as `io.BytesIO`. It needs `format`, and receives the finished workbook in one @@ -334,49 +322,40 @@ payload = buffer.getvalue() ### Arrow With `pyarrow` installed, `to_arrow()` runs the same read and hands the buffers to pyarrow zero-copy -over the Arrow C Data Interface: +over the Arrow C Data Interface, as one column-named `pyarrow.RecordBatch`: ```python -import pyarrow as pa - with open_workbook("sales.xlsb") as workbook: - array = workbook.sheets[0].to_arrow(schema) - -batch = pa.RecordBatch.from_struct_array(array) + batch = workbook.sheets[0].to_arrow(schema) ``` pyarrow owns the buffers from that point on, so the result stays valid after the workbook is closed. -#### Streaming: RecordBatchReader, pandas, and polars +#### Streaming: Arrow streams, pandas, and polars -`to_record_batch_reader()` is `to_arrow()` a batch at a time, as a streaming -`pyarrow.RecordBatchReader` — peak memory is one batch rather than one sheet: +`to_arrow_stream()` is `to_arrow()` a batch at a time — peak memory is one batch rather than one +sheet. It returns a single-use stream implementing the Arrow PyCapsule interface +(`__arrow_c_stream__`) and needs no third-party package itself; hand it to any consumer, such as +pyarrow for a `RecordBatchReader`: ```python +import pyarrow as pa + with open_workbook("sales.xlsb") as workbook: - reader = workbook.sheets[0].to_record_batch_reader(schema, batch_size=10_000) + reader = pa.RecordBatchReader.from_stream(workbook.sheets[0].to_arrow_stream(schema, batch_size=10_000)) for batch in reader: - ... + frame = batch.to_pandas() # or polars.from_arrow(batch), for one DataFrame per batch ``` -Same `batch_size` rules as `iter_parse_typed()` above, except `to_record_batch_reader()` raises -immediately rather than on the first iteration. The reader keeps working after the workbook is closed. - -`iter_pandas()`/`iter_polars()` build on it, yielding one DataFrame per batch (requires pyarrow, plus -pandas or polars respectively): - -```python -with open_workbook("sales.xlsb") as workbook: - for frame in workbook.sheets[0].iter_polars(schema, batch_size=10_000): - ... -``` +Same `batch_size` rules as `iter_parse_typed()` above, except `to_arrow_stream()` raises +immediately rather than on the first iteration. The stream keeps working after the workbook is closed. -`to_pandas()`/`to_polars()` materialize the whole sheet as a single DataFrame and also take a -`batch_size`, but the two spend it differently — do not blur them. `to_polars()` genuinely streams: -polars consumes the reader batch by batch, so the whole sheet is never resident as Arrow buffers at -once. `to_pandas()` does **not** bound peak memory the same way — `RecordBatchReader.read_all()` -concatenates every batch before pandas ever sees them. What `batch_size` buys `to_pandas()` instead is -the conversion: `self_destruct=True` frees each Arrow chunk as pandas consumes it, so the sheet is +`to_pandas()`/`to_polars()` materialize the whole sheet as a single DataFrame and both take +`batch_size` and `parallelism`, but they spend `batch_size` differently — do not blur them. +`to_polars()` genuinely streams: polars consumes the stream batch by batch, so the whole sheet is +never resident as Arrow buffers at once. `to_pandas()` does **not** bound peak memory the same way — +it concatenates every batch before pandas ever sees them. What `batch_size` buys `to_pandas()` instead +is the conversion: `self_destruct=True` frees each Arrow chunk as pandas consumes it, so the sheet is never held twice, once as Arrow and once as the DataFrame. **A faulted stream surfaces differently from every other native error in this library**: pyarrow's @@ -452,6 +431,7 @@ with open_workbook("export.csv", format="csv", options=OpenOptions(csv_delimiter ``` `csv_delimiter` and `csv_quote` are byte values, so pass `ord(";")` rather than `";"`. +`max_cell_bytes` and `intern_strings` apply to every format, a CSV included. The `max_*` fields are resource limits rather than tuning knobs: they bound what a malformed or hostile file can make the reader allocate, and exceeding one raises `ExcelReaderError`. Lower them @@ -497,7 +477,7 @@ An explicit `format="xlsx"`/`format="xlsb"` works for an encrypted file too, the `format` unset — both routes decrypt correctly given the right `password`. Writing an encrypted workbook is a second step: write the plaintext package with `write_workbook` -(or any of the `write_*` helpers), then wrap it with `encrypt_package`, which produces an +or `open_writer`, then wrap it with `encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container — AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: @@ -538,24 +518,22 @@ fastexcel 0.21.0), pandas 3.0.5 (with openpyxl 3.1.5), xlsxwriter 3.2.9. | API | Median | What it produces | |---|---:|---| -| `to_record_batch_reader()` | 31.9 ms | Arrow record batches, streamed | +| `to_arrow_stream()`, read by pyarrow | 31.9 ms | Arrow record batches, streamed | | `to_arrow()` | 31.8 ms | the same parse, handed to pyarrow zero-copy | | `parse_typed()` | 33.5 ms | typed columnar buffers, converted natively | | `iter_parse_typed()` | 34.2 ms | the same typed buffers, in batches | | `to_polars()` | 34.6 ms | typed columnar DataFrame, schema inferred | | `polars.read_excel()` | 125.6 ms | typed columnar DataFrame, types inferred | -| `read_all_columnar()` | 463.3 ms | raw columnar cells, no per-cell Python objects | | `rows()` | 1,034.6 ms | one `Cell` object per cell, streamed per row | -| `read_all()` | 1,631.0 ms | one `Cell` object per cell, all at once | The batched readers cost the same as their whole-sheet counterparts and lower the peak memory: -`to_record_batch_reader(batch_size=10000)` peaks at 11.4 MiB against 18.8 MiB for one whole-sheet +`to_arrow_stream(batch_size=10000)` peaks at 11.4 MiB against 18.8 MiB for one whole-sheet record batch on this file. Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison (~3.6x), and even that one is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules are not identical. The rows above it produce different things and are listed to show what each API -costs, not to rank them — `read_all()` is ~49x slower than `parse_typed()` because it materializes +costs, not to rank them — `rows()` is ~30x slower than `parse_typed()` because it materializes 917,504 Python objects, which is the price of that shape, not a slow parser. CSV, from `65K_Records_Data.csv` (8.2 MB), same 14 typed columns: @@ -583,14 +561,14 @@ threads](#reading-a-large-sheet-on-several-threads). | `write_workbook()` → xls | 44.4 ms | 17.7 MB | | `write_workbook()` → xlsb | 71.5 ms | 5.1 MB | | `write_workbook()` → xlsx | 90.6 ms | 5.1 MB | -| `write_polars()` → xlsx | 487.4 ms | 5.1 MB | -| `write_pandas()` → xlsx | 493.8 ms | 5.1 MB | +| `write_workbook(polars_df)` → xlsx | 487.4 ms | 5.1 MB | +| `write_workbook(pandas_df)` → xlsx | 493.8 ms | 5.1 MB | | `polars.DataFrame.write_excel()` | 4,670.4 ms | 5.6 MB | | `pandas.DataFrame.to_excel()` | 7,192.9 ms | 5.5 MB | The two DataFrame comparisons are matched work — same DataFrame in, xlsx out both times: -`write_polars()` is ~9.6x faster than polars' own `write_excel()`, and `write_pandas()` ~14.6x -faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes +`write_workbook(polars_df)` is ~9.6x faster than polars' own `write_excel()`, and +`write_workbook(pandas_df)` ~14.6x faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes through Arrow and then a Python list before reaching the native columns, which is most of the gap between the ~490 ms rows and the 90 ms one. Handing `write_workbook()` buffers that are already columnar — what `parse_typed()` returns — skips all of it. diff --git a/python/benchmarks/bench_polars_engine.py b/python/benchmarks/bench_polars_engine.py index 4e1f9b29..152a6c62 100644 --- a/python/benchmarks/bench_polars_engine.py +++ b/python/benchmarks/bench_polars_engine.py @@ -83,7 +83,7 @@ def _frame(rows: int) -> pl.DataFrame: def _generate(data_dir: Path, rows: int) -> dict[str, Path]: - from excelreader import encrypt_package, write_polars + from excelreader import encrypt_package, write_workbook data_dir.mkdir(parents=True, exist_ok=True) paths = { @@ -101,7 +101,7 @@ def _generate(data_dir: Path, rows: int) -> dict[str, Path]: if not paths["plain.xlsx"].exists(): df.write_excel(paths["plain.xlsx"], autofit=False) if not paths["plain.xlsb"].exists(): - write_polars(paths["plain.xlsb"], df) + write_workbook(paths["plain.xlsb"], df) for suffix in ("xlsx", "xlsb"): if not paths[f"encrypted.{suffix}"].exists(): encrypt_package(paths[f"plain.{suffix}"], paths[f"encrypted.{suffix}"], PASSWORD) diff --git a/python/benchmarks/bench_read.py b/python/benchmarks/bench_read.py index 9ca3f523..b848a77e 100644 --- a/python/benchmarks/bench_read.py +++ b/python/benchmarks/bench_read.py @@ -9,7 +9,7 @@ zero rows or zero cells is a harness bug, not a result — this script asserts against that instead of silently publishing a zero. -NOTE: read_all()/rows() build one Cell tuple per cell; polars.read_excel returns a typed columnar +NOTE: rows() builds one Cell tuple per cell; polars.read_excel returns a typed columnar DataFrame with type inference. That is not matched work (see STYLEGUIDE.md "Tests and Benchmarks") — this script labels the comparison, it does not pretend it is apples-to-apples. The parse_typed/parse_arrow variants are the closest thing to matched work against polars: they @@ -40,14 +40,6 @@ def _report(name: str, times: list[float], n: int) -> None: print(f"{name}: min={min(times):.4f}s median={statistics.median(times):.4f}s (n={n})") -def bench_read_all(path: Path) -> tuple[int, int]: - with excelreader.open_workbook(path) as workbook: - all_rows = workbook.sheets[0].read_all() - row_count = len(all_rows) - cell_count = sum(len(row) for row in all_rows) - return row_count, cell_count - - def bench_rows(path: Path) -> tuple[int, int]: with excelreader.open_workbook(path) as workbook: row_count = 0 @@ -58,12 +50,6 @@ def bench_rows(path: Path) -> tuple[int, int]: return row_count, cell_count -def bench_read_all_columnar(path: Path) -> tuple[int, int]: - with excelreader.open_workbook(path) as workbook: - sheet = workbook.sheets[0].read_all_columnar() - return len(sheet.row_offsets) - 1, len(sheet.columns) - - _T = excelreader.ColumnType _FIXTURE_SCHEMA = [ excelreader.ColumnSpec(type_, name=name) @@ -94,8 +80,8 @@ def bench_parse_typed(path: Path) -> tuple[int, int]: def bench_to_arrow(path: Path) -> tuple[int, int]: with excelreader.open_workbook(path) as workbook: - array = workbook.sheets[0].to_arrow(_FIXTURE_SCHEMA) - return len(array), len(array) * array.type.num_fields + batch = workbook.sheets[0].to_arrow(_FIXTURE_SCHEMA) + return batch.num_rows, batch.num_rows * batch.num_columns def bench_iter_parse_typed(path: Path) -> tuple[int, int]: @@ -107,16 +93,20 @@ def bench_iter_parse_typed(path: Path) -> tuple[int, int]: return rows, rows * len(_FIXTURE_SCHEMA) -def bench_record_batch_reader(path: Path) -> tuple[int, int]: +def _record_batch_reader(workbook): + import pyarrow + + return pyarrow.RecordBatchReader.from_stream(workbook.sheets[0].to_arrow_stream(_FIXTURE_SCHEMA, batch_size=10000)) + + +def bench_arrow_stream(path: Path) -> tuple[int, int]: rows = 0 with excelreader.open_workbook(path) as workbook: - for batch in workbook.sheets[0].to_record_batch_reader(_FIXTURE_SCHEMA, batch_size=10000): + for batch in _record_batch_reader(workbook): rows += batch.num_rows return rows, rows * len(_FIXTURE_SCHEMA) - - class _ProcessMemoryCountersEx(ctypes.Structure): _fields_ = [ ("cb", ctypes.c_uint32), @@ -149,7 +139,7 @@ def peak_leg_whole_sheet(path: Path) -> tuple[int, int]: """Whole-sheet Arrow read, sampled once while the full table is live. Returns (rows, peak).""" base = _private_bytes() with excelreader.open_workbook(path) as workbook: - batch = workbook.sheets[0].to_record_batch(_FIXTURE_SCHEMA) + batch = workbook.sheets[0].to_arrow(_FIXTURE_SCHEMA) peak = _private_bytes() - base rows = batch.num_rows return rows, peak @@ -161,7 +151,7 @@ def peak_leg_streamed(path: Path) -> tuple[int, int]: rows = 0 peak = 0 with excelreader.open_workbook(path) as workbook: - for batch in workbook.sheets[0].to_record_batch_reader(_FIXTURE_SCHEMA, batch_size=10000): + for batch in _record_batch_reader(workbook): rows += batch.num_rows peak = max(peak, _private_bytes() - base) del batch @@ -174,7 +164,7 @@ def peak_leg_streamed(path: Path) -> tuple[int, int]: def _measure_peaks(path: Path) -> dict[str, int]: """Runs each peak leg in a FRESH subprocess and collects its number. - In-process would read zero: the legs above allocate far more than these reads do (read_all alone + In-process would read zero: the legs above allocate far more than these reads do (rows() alone builds ~900k Cell tuples), a heap is not handed back to the OS on free, and so by the time we got here the growth we are trying to see would already be absorbed by slack. """ @@ -313,18 +303,10 @@ def main() -> int: print(f"file: {args.path}") print() - print("excelreader.read_all():") - _time_and_assert(" read_all", bench_read_all, args.path, args.n) - print() - print("excelreader.rows() (per-row iteration):") _time_and_assert(" rows", bench_rows, args.path, args.n) print() - print("excelreader.read_all_columnar() [no per-cell Cell/str objects — the fast path]:") - _time_and_assert(" read_all_columnar", bench_read_all_columnar, args.path, args.n) - print() - if args.path.resolve() == DEFAULT_PATH.resolve(): print("parse_typed() [schema-driven typed columns, native-side conversion]:") _time_and_assert(" parse_typed", bench_parse_typed, args.path, args.n) @@ -344,12 +326,12 @@ def main() -> int: _time_and_assert(" to_arrow", bench_to_arrow, args.path, args.n) print() - print("excelreader.to_record_batch_reader() (batched):") - _time_and_assert(" record_batch_reader", bench_record_batch_reader, args.path, args.n) + print("excelreader.to_arrow_stream() read by pyarrow (batched):") + _time_and_assert(" arrow_stream", bench_arrow_stream, args.path, args.n) print() if sys.platform == "win32": - print("peak memory [to_record_batch whole sheet vs to_record_batch_reader(batch_size=10000)]:") + print("peak memory [to_arrow whole sheet vs to_arrow_stream(batch_size=10000)]:") peaks = _measure_peaks(args.path) saved = peaks["whole_sheet"] - peaks["streamed"] print(f" streaming holds {saved} bytes ({saved / 1024 / 1024:.1f} MiB) less at the peak") diff --git a/python/benchmarks/bench_write.py b/python/benchmarks/bench_write.py index 00c14a90..620aa490 100644 --- a/python/benchmarks/bench_write.py +++ b/python/benchmarks/bench_write.py @@ -76,16 +76,16 @@ def run() -> None: def benchmark_pandas(args: argparse.Namespace, tmp: Path): pd = find_spec("pandas") if pd is None: - print("pandas not installed — skipping write_pandas/to_excel comparison") + print("pandas not installed — skipping the pandas write/to_excel comparison") print() return with excelreader.open_workbook(args.path, format="xlsb") as workbook: pandas_df = workbook.sheets[0].to_pandas(_FIXTURE_SCHEMA) - print("write_pandas() [pandas.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") + print("write_workbook(pandas) [pandas.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") _time_and_assert( - " write_pandas", - lambda p: excelreader.write_pandas(p, pandas_df), + " write_workbook(pandas)", + lambda p: excelreader.write_workbook(p, pandas_df), tmp / "out_pandas.xlsx", args.n, ) @@ -93,7 +93,7 @@ def benchmark_pandas(args: argparse.Namespace, tmp: Path): openpyxl = find_spec("openpyxl") if openpyxl is None: - print("openpyxl not installed — skipping write_pandas/write_workbook comparison") + print("openpyxl not installed — skipping the pandas.DataFrame.to_excel() comparison") print() return print("pandas.DataFrame.to_excel() [same DataFrame, matched work — both write xlsx]:") @@ -108,17 +108,17 @@ def benchmark_pandas(args: argparse.Namespace, tmp: Path): def benchmark_polars(args: argparse.Namespace, tmp: Path): pl = find_spec("polars") if pl is None: - print("polars not installed — skipping write_polars/write_excel comparison") + print("polars not installed — skipping the polars write/write_excel comparison") print() return with excelreader.open_workbook(args.path, format="xlsb") as workbook: schema = workbook.sheets[0].infer_schema(sample_size=10) polars_df = workbook.sheets[0].to_polars(schema) - print("write_polars() [polars.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") + print("write_workbook(polars) [polars.DataFrame -> Arrow -> pylist -> native columns -> xlsx]:") _time_and_assert( - " write_polars", - lambda p: excelreader.write_polars(p, polars_df), + " write_workbook(polars)", + lambda p: excelreader.write_workbook(p, polars_df), tmp / "out_polars.xlsx", args.n, ) @@ -199,7 +199,7 @@ def main() -> int: try: import pyarrow # noqa: F401 except ImportError: - print("pyarrow not installed — skipping write_pandas/write_polars/pandas/polars comparisons") + print("pyarrow not installed — skipping the DataFrame writes and pandas/polars comparisons") return 0 benchmark_pandas(args, tmp) diff --git a/python/examples/read_workbook.py b/python/examples/read_workbook.py index d12fa407..46ffd727 100644 --- a/python/examples/read_workbook.py +++ b/python/examples/read_workbook.py @@ -14,7 +14,7 @@ def main() -> int: max_rows = int(sys.argv[2]) if len(sys.argv) > 2 else 10 with open_workbook(path) as workbook: - print(f"sheets={workbook.sheet_names} date1904={workbook.is_date1904}") + print(f"sheets={[sheet.name for sheet in workbook.sheets]} date1904={workbook.is_date1904}") for index, row in enumerate(workbook.sheets[0].rows()): if index >= max_rows: break diff --git a/python/src/excelreader/__init__.py b/python/src/excelreader/__init__.py index 826fc891..ed6e12f4 100644 --- a/python/src/excelreader/__init__.py +++ b/python/src/excelreader/__init__.py @@ -1,12 +1,10 @@ """Python bindings for ExcelReader.""" -from excelreader.arrow_stream import ArrowStream -from excelreader.reader import Sheet, Sheets, Workbook, decode_cell, open_workbook +from excelreader.reader import Sheet, Sheets, Workbook, open_workbook from excelreader.sources import Source from excelreader.types import ( Cell, CellType, - ColumnarSheet, ColumnSpec, ColumnType, ExcelReaderError, @@ -18,22 +16,14 @@ TypedTable, WriteOptions, ) -from excelreader.writer import ( - encrypt_package, - write_arrow, - write_pandas, - write_polars, - write_workbook, -) +from excelreader.writer import encrypt_package, write_workbook from excelreader.writer_handle import SheetWriter, open_writer __all__ = [ - "ArrowStream", "Cell", "CellType", "ColumnSpec", "ColumnType", - "ColumnarSheet", "ExcelReaderError", "OpenOptions", "PasswordIncorrectError", @@ -47,12 +37,8 @@ "TypedTable", "Workbook", "WriteOptions", - "decode_cell", "encrypt_package", "open_workbook", "open_writer", - "write_arrow", - "write_pandas", - "write_polars", "write_workbook", ] diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index cf8f5b25..e39b2e50 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -60,22 +60,6 @@ XL_T_TIMESTAMP = 6 -class NativeRowCell(ctypes.Structure): - _fields_ = [ - ("column", ctypes.c_int32), - ("type", ctypes.c_int32), - ("value_len", ctypes.c_int32), - ("value", ctypes.POINTER(ctypes.c_uint8)), - ] - - -class NativeRow(ctypes.Structure): - _fields_ = [ - ("cell_count", ctypes.c_int32), - ("cells", ctypes.POINTER(NativeRowCell)), - ] - - class NativeColumnSpec(ctypes.Structure): """Mirrors xl_column_spec. `names`/`name_lens`/`name_count` may be left NULL/NULL/0 to resolve by `index` instead. Build one with `column_spec_by_name`/`column_spec_by_names`, never directly — @@ -225,15 +209,6 @@ class ArrowArrayStream(ctypes.Structure): ] -class NativeRows(ctypes.Structure): - """Mirrors xl_rows_decoded.""" - - _fields_ = [ - ("row_count", ctypes.c_int32), - ("rows", ctypes.POINTER(NativeRow)), - ] - - class NativeBuffer(ctypes.Structure): """Mirrors xl_buffer.""" @@ -332,13 +307,14 @@ def to_native_open_options(options: OpenOptions) -> NativeOpenOptions: raw = default_open_options() raw.csv_sniff_dialect = _opt_state(options.csv_sniff_dialect) raw.csv_detect_bom = _opt_state(options.csv_detect_bom) - raw.csv_intern_strings = _opt_state(options.csv_intern_strings) raw.prefetch_decompression = _opt_state(options.prefetch_decompression) raw.intern_strings = _opt_state(options.intern_strings) raw.verify_zip_checksums = _opt_state(options.verify_zip_checksums) raw.csv_delimiter = _opt_number(options.csv_delimiter) raw.csv_quote = _opt_number(options.csv_quote) - raw.csv_max_cell_bytes = _opt_number(options.csv_max_cell_bytes) + # The ABI's csv_* copies win over these for a CSV even when unset, so send the same value to both. + raw.csv_max_cell_bytes = _opt_number(options.max_cell_bytes) + raw.csv_intern_strings = raw.intern_strings raw.max_total_decompressed_bytes = _opt_number(options.max_total_decompressed_bytes) raw.max_cell_bytes = _opt_number(options.max_cell_bytes) raw.max_shared_string_bytes = _opt_number(options.max_shared_string_bytes) @@ -495,12 +471,6 @@ def _bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.xl_rows_close.restype = c_int lib.xl_rows_next.argtypes = [p_void, p_bytes, c_int, p_int] lib.xl_rows_next.restype = c_int - lib.xl_rows_read_all_blob.argtypes = [p_void, p_bytes, c_int, p_int] - lib.xl_rows_read_all_blob.restype = c_int - lib.xl_rows_read_all_decoded.argtypes = [p_void, ctypes.POINTER(NativeRows)] - lib.xl_rows_read_all_decoded.restype = c_int - lib.xl_free_rows.argtypes = [ctypes.POINTER(NativeRows)] - lib.xl_free_rows.restype = None lib.xl_parse_typed.argtypes = [p_void, c_int, p_specs, c_int, c_int, c_int, ctypes.POINTER(NativeTable)] lib.xl_parse_typed.restype = c_int lib.xl_typed_reader_open.argtypes = [p_void, c_int, p_specs, c_int, c_int, ctypes.c_int64, pp_void] diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index 0790a2b4..118d074e 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -23,7 +23,6 @@ from excelreader.types import ( Cell, CellType, - ColumnarSheet, ColumnSpec, ColumnType, ExcelReaderError, @@ -44,7 +43,6 @@ _CELL_HEADER = struct.Struct(" str: @@ -109,24 +107,21 @@ def _require_handle(self) -> ctypes.c_void_p: raise ExcelReaderError("workbook is closed") return self._handle - @property - def sheet_count(self) -> int: + def _sheet_count(self) -> int: count = ctypes.c_int32() _check(self._lib.xl_sheet_count(self._require_handle(), ctypes.byref(count))) return count.value - def sheet_name_at(self, index: int) -> str: + def _sheet_name_at(self, index: int) -> str: raw, length = _fill_buffer(self._lib.xl_sheet_name_at, self._require_handle(), index, initial=256) return raw[:length].decode("utf-8") - @property - def sheet_names(self) -> list[str]: - """Every sheet's name, in order.""" - return [self.sheet_name_at(index) for index in range(self.sheet_count)] - @property def sheets(self) -> Sheets: - """The workbook's sheets: `len()`, iteration, and lookup by index, slice or name.""" + """The workbook's sheets: `len()`, iteration, and lookup by index, slice or name. + + `[sheet.name for sheet in workbook.sheets]` lists every sheet's name. + """ return Sheets(self) @property @@ -136,7 +131,7 @@ def is_date1904(self) -> bool: return flag.value != 0 def close(self) -> None: - """Closes the workbook. A generator that has already started, and any record-batch reader already opened, keep working.""" + """Closes the workbook. A generator that has already started, and any Arrow stream already opened, keep working.""" with self._close_lock: handle, self._handle = self._handle, None if handle is None: @@ -163,7 +158,7 @@ def __init__(self, workbook: Workbook) -> None: self._workbook = workbook def __len__(self) -> int: - return self._workbook.sheet_count + return self._workbook._sheet_count() @overload def __getitem__(self, key: SupportsIndex | str) -> Sheet: ... @@ -219,7 +214,7 @@ def index(self) -> int: @property def name(self) -> str: - return self._workbook.sheet_name_at(self._index) + return self._workbook._sheet_name_at(self._index) @property def visibility(self) -> SheetVisibility: @@ -257,41 +252,11 @@ def rows(self) -> Iterator[list[Cell]]: finally: self._close_rows(cursor) - def read_all(self) -> list[list[Cell]]: - """Materializes every row of this sheet in one native call. - - Builds one `Cell`/`str` object per cell — for a large sheet, `read_all_columnar()` is - substantially faster because it never does that. - """ - cursor = self._open_rows() - try: - rows = _native.NativeRows() - _check(self._lib.xl_rows_read_all_decoded(cursor, ctypes.byref(rows))) - try: - return [_decode_native_row(rows.rows[index]) for index in range(rows.row_count)] - finally: - self._lib.xl_free_rows(ctypes.byref(rows)) - finally: - self._close_rows(cursor) - - def read_all_columnar(self) -> ColumnarSheet: - """Materializes every row of this sheet as parallel flat arrays. - - The fast path for large sheets: unlike `read_all()`, this never constructs a `Cell`/`str` - per cell. Use `decode_cell()` to materialize one cell on demand. - """ - cursor = self._open_rows() - try: - raw, written = _fill_buffer(self._lib.xl_rows_read_all_blob, cursor, initial=_INITIAL_ALL_ROWS_BUFFER) - finally: - self._close_rows(cursor) - return _decode_columnar(raw, written) - def parse_typed(self, schema: Sequence[ColumnSpec], header_row: int = 1, parallelism: int = 1) -> TypedTable: """Reads this whole sheet into typed columns, converting values on the native side. - By far the fastest path in this library: unlike every other read method, no cell is ever - formatted to text, so a large sheet costs a fraction of `read_all_columnar()`. + By far the fastest path in this library: unlike `rows()`, no cell is ever formatted to text + or turned into a Python object. `header_row` is a 1-based row number whose values name the columns (that row is skipped and never yielded as data); 0 means the sheet has no header, in which case every spec must @@ -321,7 +286,7 @@ def iter_parse_typed( the workbook is closed. Being a generator, it opens nothing until the first iteration, so a bad `batch_size` is not - reported until then; `to_record_batch_reader()` raises immediately instead. + reported until then; `to_arrow_stream()` raises immediately instead. """ handle = self._workbook._require_handle() specs = _build_specs(schema) @@ -346,57 +311,23 @@ def iter_parse_typed( self._lib.xl_typed_reader_close(reader) def to_arrow(self, schema: Sequence[ColumnSpec], header_row: int = 1, parallelism: int = 1) -> object: - """The same read as `parse_typed()`, handed to pyarrow as one `StructArray`, zero-copy. + """The same read as `parse_typed()`, handed to pyarrow as one column-named `RecordBatch`, zero-copy. - Requires pyarrow. The returned array owns the native buffers through the Arrow C Data - Interface's release callback, so it stays valid after this workbook is closed. Wrap it with - `pyarrow.RecordBatch.from_struct_array()` for a column-named batch. `parallelism` works as in - `parse_typed()`. + Requires pyarrow. The batch owns the native buffers through the Arrow C Data Interface's + release callback, so it stays valid after this workbook is closed. `parallelism` works as in + `parse_typed()`. To stream instead, see `to_arrow_stream()`. """ - try: - import pyarrow - except ImportError: - raise ImportError( - "to_arrow() requires pyarrow — install it with `pip install pyarrow`, or use " - "parse_typed(), which returns the same data with no third-party dependency." - ) from None - + pyarrow = _import_pyarrow("to_arrow()", "parse_typed()") handle = self._workbook._require_handle() specs = _build_specs(schema) array = _native.ArrowArray() arrow_schema = _native.ArrowSchema() _check(self._lib.xl_parse_arrow(handle, self._index, specs, len(specs), header_row, parallelism, ctypes.byref(array), ctypes.byref(arrow_schema))) - return pyarrow.Array._import_from_c(ctypes.addressof(array), ctypes.addressof(arrow_schema)) - - def to_record_batch(self, schema: Sequence[ColumnSpec], header_row: int = 1, parallelism: int = 1) -> object: - """Same read as `to_arrow()`, wrapped as a column-named `pyarrow.RecordBatch`. - - Requires pyarrow — see `to_arrow()`. - """ - array = self.to_arrow(schema, header_row=header_row, parallelism=parallelism) - import pyarrow - - return pyarrow.RecordBatch.from_struct_array(array) - - def to_record_batch_reader( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 - ) -> object: - """The same read as `to_arrow()`, as a streaming `pyarrow.RecordBatchReader`. - - Peak memory is one batch rather than one sheet. Requires pyarrow. - - pyarrow owns the stream once it is imported — exhausting or dropping the reader is what - closes the underlying read. - """ - try: - import pyarrow - except ImportError: - raise ImportError( - "to_record_batch_reader() requires pyarrow — install it with `pip install " - "pyarrow`, or use iter_parse_typed(), which streams the same data with no " - "third-party dependency." - ) from None + return pyarrow.RecordBatch._import_from_c(ctypes.addressof(array), ctypes.addressof(arrow_schema)) + def _open_arrow_stream( + self, schema: Sequence[ColumnSpec], header_row: int, batch_size: int + ) -> _native.ArrowArrayStream: handle = self._workbook._require_handle() specs = _build_specs(schema) stream = _native.ArrowArrayStream() @@ -405,75 +336,37 @@ def to_record_batch_reader( handle, self._index, specs, len(specs), header_row, batch_size, ctypes.byref(stream) ) ) - return pyarrow.RecordBatchReader._import_from_c(ctypes.addressof(stream)) + return stream def to_arrow_stream( self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 ) -> ArrowStream: - """The same read as `to_record_batch_reader()`, as a single-use Arrow PyCapsule stream. - - Needs no third-party package to create; pass it to any consumer of `__arrow_c_stream__` - (`polars.DataFrame(stream)`, `pyarrow.RecordBatchReader.from_stream(stream)`, ...). - """ - handle = self._workbook._require_handle() - specs = _build_specs(schema) - stream = _native.ArrowArrayStream() - _check( - self._lib.xl_parse_arrow_stream( - handle, self._index, specs, len(specs), header_row, batch_size, ctypes.byref(stream) - ) - ) - return ArrowStream(stream) - - def iter_pandas( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 - ) -> Iterator[object]: - """One `pandas.DataFrame` per batch, streamed. Requires pyarrow and pandas. + """The same read as `to_arrow()`, streamed `batch_size` rows at a time as a single-use Arrow + PyCapsule stream. Peak memory is one batch rather than one sheet. - A generator, so a missing dependency is reported at the first iteration, not at the call. + Needs no third-party package to create; pass it to any consumer of `__arrow_c_stream__`: + `polars.DataFrame(stream)`, or `pyarrow.RecordBatchReader.from_stream(stream)` for a + `RecordBatchReader`. The stream stays valid after this workbook is closed. """ - try: - import pyarrow # noqa: F401 - except ImportError: - raise ImportError( - "iter_pandas() requires pyarrow — install it with `pip install pyarrow`, or use " - "iter_parse_typed(), which streams the same data with no third-party dependency." - ) from None - - reader = self.to_record_batch_reader(schema, header_row=header_row, batch_size=batch_size) - for batch in reader: - yield batch.to_pandas(self_destruct=True, split_blocks=True) - - def iter_polars( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 - ) -> Iterator[object]: - """One `polars.DataFrame` per batch, streamed. Requires pyarrow and polars. - - A generator, so a missing dependency is reported at the first iteration, not at the call. - """ - try: - import polars - except ImportError: - raise ImportError( - "iter_polars() requires polars — install it with `pip install polars`, or use " - "to_record_batch_reader(), which streams the same data with no polars dependency." - ) from None - - reader = self.to_record_batch_reader(schema, header_row=header_row, batch_size=batch_size) - for batch in reader: - yield polars.from_arrow(batch, rechunk=False) + return ArrowStream(self._open_arrow_stream(schema, header_row, batch_size)) def to_pandas( - self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000 + self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000, parallelism: int = 1 ) -> object: - """Same read as `to_arrow()`, materialized as a `pandas.DataFrame`. + """Same read as `to_arrow()`, materialized as a `pandas.DataFrame`. Requires pyarrow and pandas. - `read_all()` concatenates every batch, so this does NOT bound peak memory the way - `iter_pandas()` does. What it buys is the conversion: `self_destruct` frees each Arrow - chunk as pandas takes it, so the sheet is never held twice. Requires pyarrow and pandas. + Streams `batch_size` rows at a time, then concatenates them, so peak memory is not bounded + by the batch size; `self_destruct` frees each Arrow chunk as pandas takes it, so the sheet + is never held twice. With `parallelism` other than 1 the sheet is parsed whole, as in + `parse_typed()`, instead of streamed. """ - reader = self.to_record_batch_reader(schema, header_row=header_row, batch_size=batch_size) - return reader.read_all().to_pandas(self_destruct=True, split_blocks=True) + pyarrow = _import_pyarrow("to_pandas()", "parse_typed()") + if parallelism != 1: + data = self.to_arrow(schema, header_row=header_row, parallelism=parallelism) + else: + stream = self._open_arrow_stream(schema, header_row, batch_size) + data = pyarrow.RecordBatchReader._import_from_c(ctypes.addressof(stream)).read_all() + return data.to_pandas(self_destruct=True, split_blocks=True) def to_polars( self, schema: Sequence[ColumnSpec], header_row: int = 1, batch_size: int = 10000, parallelism: int = 1 @@ -494,9 +387,8 @@ def to_polars( ) from None if parallelism != 1: - return polars.from_arrow(self.to_record_batch(schema, header_row=header_row, parallelism=parallelism)) - stream = self.to_arrow_stream(schema, header_row=header_row, batch_size=batch_size) - return polars.DataFrame(stream) + return polars.from_arrow(self.to_arrow(schema, header_row=header_row, parallelism=parallelism)) + return polars.DataFrame(self.to_arrow_stream(schema, header_row=header_row, batch_size=batch_size)) def infer_schema(self, header_row: int = 1, sample_size: int = 100, parse_text: bool = False) -> list[ColumnSpec]: """Guesses a `parse_typed()`/`to_arrow()` schema by sampling this sheet's cells. @@ -541,58 +433,15 @@ def _decode_row(blob: bytes, length: int) -> list[Cell]: return cells -def _decode_native_row(row: _native.NativeRow) -> list[Cell]: - cells: list[Cell] = [] - for index in range(row.cell_count): - cell = row.cells[index] - value = ctypes.string_at(cell.value, cell.value_len).decode("utf-8") - cells.append(Cell(column=cell.column, type=CellType(cell.type), value=value)) - return cells - - -def _decode_columnar(blob: bytes, length: int) -> ColumnarSheet: - # expensive. (ponytail: if this loop measurably dominates for very wide/tall sheets, revisit with - row_offsets = array("i", [0]) - columns = array("i") - types = array("i") - value_offsets = array("i", [0]) - values = bytearray() - - row_count = struct.unpack_from(" object: + try: + import pyarrow + except ImportError: + raise ImportError( + f"{method} requires pyarrow — install it with `pip install pyarrow`, or use " + f"{alternative}, which returns the same data with no third-party dependency." + ) from None + return pyarrow def _decode_inferred_schema(schema: _native.NativeInferredSchema) -> list[ColumnSpec]: @@ -675,19 +524,6 @@ def _decode_validity(column: _native.NativeColumn, row_count: int) -> bytes | No return ctypes.string_at(column.validity, (row_count + 7) // 8) -def _to_columnar_array(values: array) -> object: - if _numpy is None: - return values - return _numpy.frombuffer(values, dtype=_numpy.int32) - - -def decode_cell(sheet: ColumnarSheet, index: int) -> Cell: - """Materializes the `Cell` at flat cell index `index` in `sheet`, decoding only that one value.""" - start, end = int(sheet.value_offsets[index]), int(sheet.value_offsets[index + 1]) - value = bytes(sheet.values[start:end]).decode("utf-8") - return Cell(column=int(sheet.columns[index]), type=CellType(int(sheet.types[index])), value=value) - - def _raw_options( options: OpenOptions | None, password: str | None ) -> tuple[object, _native.NativeOpenOptions | None, ctypes.Array[ctypes.c_char] | None]: diff --git a/python/src/excelreader/types.py b/python/src/excelreader/types.py index c41eaf97..7319ce68 100644 --- a/python/src/excelreader/types.py +++ b/python/src/excelreader/types.py @@ -63,31 +63,6 @@ def as_date(self, date1904: bool = False) -> datetime.date | None: return datetime.date(1899, 12, 30) + datetime.timedelta(days=days) -class ColumnarSheet(NamedTuple): - """Every cell of a sheet as parallel flat arrays, produced by `Sheet.read_all_columnar()`. - - The fast path for large sheets: unlike `Sheet.read_all()`, decoding never constructs one - `Cell`/`str` object per cell — only `int` values fill the index arrays below, and `values` holds - every cell's raw UTF-8 bytes concatenated. Use `decode_cell()` to materialize a single `Cell` on - demand, or slice `values` directly for bulk work. - - Each array is `array.array('i')` normally, or a NumPy `int32` `ndarray` when NumPy is installed - (see `reader.py`) — both support the same `len()`/indexing/slicing used here. - - - `row_offsets[i] : row_offsets[i + 1]` are the cell indices belonging to row `i`, indexing into - `columns`/`types`/`value_offsets`. Length `row_count + 1`. - - `columns[j]` / `types[j]` are cell `j`'s column index and `CellType` value. Length `cell_count`. - - `value_offsets[j] : value_offsets[j + 1]` is cell `j`'s byte slice into `values`. Length - `cell_count + 1`. - """ - - row_offsets: Any - columns: Any - types: Any - value_offsets: Any - values: bytes - - class ColumnType(IntEnum): """A column's requested type in a `Sheet.parse_typed()`/`to_arrow()` schema. @@ -127,7 +102,8 @@ class OpenOptions(NamedTuple): because several of them default to true. Neither leaks up here. The `max_*` fields are resource limits, not tuning knobs — they bound what a malformed or hostile - file can make the reader allocate. Lower them when parsing untrusted uploads. + file can make the reader allocate. Lower them when parsing untrusted uploads. `max_cell_bytes` + and `intern_strings` apply to every format, a CSV included. `csv_delimiter` and `csv_quote` are byte values, not strings: pass `ord(';')`, not `';'`. @@ -149,8 +125,6 @@ class OpenOptions(NamedTuple): csv_delimiter: int | None = None csv_quote: int | None = None csv_detect_bom: bool | None = None - csv_max_cell_bytes: int | None = None - csv_intern_strings: bool | None = None max_total_decompressed_bytes: int | None = None max_cell_bytes: int | None = None max_shared_string_bytes: int | None = None @@ -188,7 +162,7 @@ class StringColumn: """A `ColumnType.STRING` column, held as UTF-8 bytes plus offsets rather than one `str` per row. Decoding is deferred to `column[i]`, so reading a million-row string column costs no Python - objects until something actually asks for a value — the same trade `ColumnarSheet` makes. + objects until something actually asks for a value. """ __slots__ = ("_data", "_offsets") diff --git a/python/src/excelreader/writer.py b/python/src/excelreader/writer.py index d7cd6740..18de080f 100644 --- a/python/src/excelreader/writer.py +++ b/python/src/excelreader/writer.py @@ -13,6 +13,7 @@ import datetime import io import os +import sys from array import array from pathlib import Path from typing import IO, Any, Union @@ -128,28 +129,40 @@ def _marshal( def write_workbook( destination: Destination, - table: TypedTable, - types: list[ColumnType], + data: Any, + types: list[ColumnType] | None = None, *, format: str | None = None, options: WriteOptions | None = None, ) -> None: - """Writes `table` as a single sheet to `destination`: a path, or a binary object with `write()` + """Writes `data` as a single sheet to `destination`: a path, or a binary object with `write()` such as `io.BytesIO` or an open file. - `table` is what `Sheet.parse_typed()` returns, so reading a sheet and writing it back out is a - round-trip through the same buffers. `types` gives each column's `ColumnType` and is required - positionally: a `TypedTable` carries raw buffers (an `array`/`StringColumn`/NumPy array), and - nothing about a raw buffer's element size alone distinguishes, say, I64 from TIME — both are - 8-byte-per-row arrays. The type tag has to travel with the call, not be guessed from the data. + `data` is one of: + + - a `TypedTable`, what `Sheet.parse_typed()` returns, so reading a sheet and writing it back out + is a round-trip through the same buffers. `types` then gives each column's `ColumnType` and is + required: a `TypedTable` carries raw buffers, and nothing about a raw buffer's element size + alone distinguishes, say, I64 from TIME — both are 8-byte-per-row arrays. + - a `pyarrow.RecordBatch` or `pyarrow.Table`, a `pandas.DataFrame` (its index is not written), + or any other table exposing `__arrow_c_stream__`, such as a `polars.DataFrame`. Each column's + type comes from its Arrow type, so `types` must be None. Requires pyarrow. `format` is one of xlsx/xlsb/xls/csv. For a path, None infers it from the extension; an object with `write()` has no extension, so it needs `format`. The whole workbook is built in memory and handed to `write()` in one call, at the object's current position; the object is not closed. - `table.names` becomes the header row. `options` sets the sheet name and CSV dialect; see + The column names become the header row. `options` sets the sheet name and CSV dialect; see `WriteOptions`. """ + if isinstance(data, TypedTable): + if types is None: + raise TypeError("types is required when data is a TypedTable: one ColumnType per column") + table = data + else: + if types is not None: + raise TypeError("types must be None unless data is a TypedTable; Arrow data carries its own types") + table, types = _table_from_arrow(_arrow_table(data)) if isinstance(destination, (str, os.PathLike)): _write_file(Path(destination), table, types, format, options) return @@ -206,6 +219,7 @@ def _write_memory( _ARROW_TYPES = { "string": ColumnType.STRING, "large_string": ColumnType.STRING, + "string_view": ColumnType.STRING, "int64": ColumnType.I64, "int32": ColumnType.I64, "double": ColumnType.F64, @@ -325,35 +339,26 @@ def _table_from_arrow(batch: Any) -> tuple[TypedTable, list[ColumnType]]: return table, types -def _arrow_from_pandas(df: Any) -> Any: - import pyarrow - - return pyarrow.Table.from_pandas(df, preserve_index=False) - - -def write_arrow( - destination: Destination, batch: Any, *, format: str | None = None, options: WriteOptions | None = None -) -> None: - """Writes a `pyarrow.RecordBatch` (or `Table`) to `destination`, as `write_workbook()` does. - Requires pyarrow.""" - table, types = _table_from_arrow(batch) - write_workbook(destination, table, types, format=format, options=options) - - -def write_pandas( - destination: Destination, df: Any, *, format: str | None = None, options: WriteOptions | None = None -) -> None: - """Writes a `pandas.DataFrame` to `destination`, as `write_workbook()` does. Requires pyarrow and - pandas.""" - write_arrow(destination, _arrow_from_pandas(df), format=format, options=options) - +def _arrow_table(data: Any) -> Any: + try: + import pyarrow + except ImportError: + raise ImportError( + "write_workbook() requires pyarrow to write Arrow or DataFrame data — install it with " + "`pip install pyarrow`, or pass a TypedTable and its types." + ) from None -def write_polars( - destination: Destination, df: Any, *, format: str | None = None, options: WriteOptions | None = None -) -> None: - """Writes a `polars.DataFrame` to `destination`, as `write_workbook()` does. Requires pyarrow and - polars.""" - write_arrow(destination, df.to_arrow(), format=format, options=options) + if isinstance(data, (pyarrow.RecordBatch, pyarrow.Table)): + return data + pandas = sys.modules.get("pandas") + if pandas is not None and isinstance(data, pandas.DataFrame): + return pyarrow.Table.from_pandas(data, preserve_index=False) + if hasattr(data, "__arrow_c_stream__"): + return pyarrow.RecordBatchReader._import_from_c_capsule(data.__arrow_c_stream__()).read_all() + raise TypeError( + "data must be a TypedTable, a pyarrow RecordBatch or Table, a pandas DataFrame, or an object " + f"with __arrow_c_stream__; got {type(data).__name__}" + ) def encrypt_package( diff --git a/python/src/excelreader/writer_handle.py b/python/src/excelreader/writer_handle.py index 7bfd1c07..a047ac05 100644 --- a/python/src/excelreader/writer_handle.py +++ b/python/src/excelreader/writer_handle.py @@ -34,13 +34,17 @@ def _is_writable(value: Any) -> bool: ) +def _is_typed_blank(value: Any) -> bool: + return type(value) is tuple and len(value) == 2 and value[0] is None and isinstance(value[1], ColumnType) + + class SheetWriter: """A streaming workbook writer. Use as a context manager; closing finalizes the workbook, and for a destination with `write()` hands it the finished bytes. - The required call order is `start_sheet` -> `start_row` -> one write per cell -> `end_row` -> - `end_sheet`. Calling out of order raises `ExcelReaderError`, and the writer stays usable: fix - the order and continue, or close to discard. + The required call order is `start_sheet` -> `write_row` per row -> `end_sheet`. Calling out of + order raises `ExcelReaderError`, and the writer stays usable: fix the order and continue, or + close to discard. """ __slots__ = ("_handle", "_lib", "_destination") @@ -59,115 +63,60 @@ def start_sheet(self, name: str) -> None: pointer, length = _encode(name) _check(self._lib.xl_start_sheet(self._require_handle(), pointer, length)) - def start_row(self) -> None: - _check(self._lib.xl_start_row(self._require_handle())) - - def end_row(self) -> None: - _check(self._lib.xl_end_row(self._require_handle())) - def end_sheet(self) -> None: _check(self._lib.xl_end_sheet(self._require_handle())) - def write_str(self, value: str | None) -> None: - """Writes the next cell as text, or a blank cell for None.""" - handle = self._require_handle() - if value is None: - _check(self._lib.xl_write_string(handle, None, 0)) - return - pointer, length = _encode(value) - _check(self._lib.xl_write_string(handle, pointer, length)) - - def write_i64(self, value: int | None) -> None: - """Writes the next cell as a whole number. Raises `OverflowError` outside the signed 64-bit range.""" - if value is None: - self.write_null(ColumnType.I64) - return - # ctypes would keep the low 64 bits of a larger value without complaint. - if not _INT64_MIN <= value <= _INT64_MAX: - raise OverflowError(f"{value} does not fit a 64-bit integer cell; write it with write_f64 or write_str") - _check(self._lib.xl_write_int64(self._require_handle(), value)) - - def write_f64(self, value: float | None) -> None: - if value is None: - self.write_null(ColumnType.F64) - return - _check(self._lib.xl_write_float64(self._require_handle(), value)) - - def write_bool(self, value: bool | None) -> None: - if value is None: - self.write_null(ColumnType.BOOL) - return - _check(self._lib.xl_write_bool(self._require_handle(), 1 if value else 0)) - - def write_date(self, value: datetime.date | None) -> None: - if value is None: - self.write_null(ColumnType.DATE) - return - _check(self._lib.xl_write_date(self._require_handle(), (value - _EPOCH_DATE).days)) - - def write_time(self, value: datetime.time | None) -> None: - if value is None: - self.write_null(ColumnType.TIME) - return - micros = ( - (value.hour * 3600 + value.minute * 60 + value.second) * 1_000_000 + value.microsecond - ) - _check(self._lib.xl_write_time(self._require_handle(), micros)) - - def write_timestamp(self, value: datetime.datetime | None) -> None: - if value is None: - self.write_null(ColumnType.TIMESTAMP) - return - if value.tzinfo is not None: - value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None) - delta = value - _EPOCH_DATETIME - micros = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds - _check(self._lib.xl_write_timestamp(self._require_handle(), micros)) - - def write_null(self, column_type: ColumnType) -> None: - """Writes a blank cell typed as `column_type`.""" - _check(self._lib.xl_write_null(self._require_handle(), int(column_type))) - def write_row(self, values: Sequence[Any]) -> None: - """`start_row()`, one inferred write per value, `end_row()`. + """Writes one row, one cell per value, its type inferred from the value. - The type of each value picks the write method. `None` becomes a blank *string* cell — it is - the one case inference cannot resolve; use `write_null(ColumnType.X)` for a typed blank. + `str` is text, `bool` a boolean, `int` a 64-bit integer, `float` a number, `datetime` a + timestamp (an aware one converted to UTC), `date` a date and `time` a time of day. `None` is a + blank text cell; the tuple `(None, ColumnType.X)` is a blank cell typed as X. Raises, before writing anything, `TypeError` for a value whose type has no mapping and `OverflowError` for an `int` outside the signed 64-bit range. """ for position, value in enumerate(values): - if not _is_writable(value): + if not _is_writable(value) and not _is_typed_blank(value): raise TypeError( f"write_row cannot infer a cell type for {type(value).__name__} " f"at position {position}" ) + # ctypes would keep the low 64 bits of a larger value without complaint. if type(value) is int and not _INT64_MIN <= value <= _INT64_MAX: raise OverflowError(f"{value} at position {position} does not fit a 64-bit integer cell") - self.start_row() + handle = self._require_handle() + _check(self._lib.xl_start_row(handle)) for value in values: - self._write_inferred(value) - self.end_row() + _check(self._write_cell(handle, value)) + _check(self._lib.xl_end_row(handle)) - def _write_inferred(self, value: Any) -> None: + def _write_cell(self, handle: ctypes.c_void_p, value: Any) -> int: + lib = self._lib if value is None: - self.write_str(None) - elif isinstance(value, bool): - self.write_bool(value) - elif isinstance(value, int): - self.write_i64(value) - elif isinstance(value, float): - self.write_f64(value) - elif isinstance(value, datetime.datetime): - self.write_timestamp(value) - elif isinstance(value, datetime.date): - self.write_date(value) - elif isinstance(value, datetime.time): - self.write_time(value) - else: - self.write_str(value) + return lib.xl_write_string(handle, None, 0) + if isinstance(value, tuple): + return lib.xl_write_null(handle, int(value[1])) + if isinstance(value, bool): + return lib.xl_write_bool(handle, 1 if value else 0) + if isinstance(value, int): + return lib.xl_write_int64(handle, value) + if isinstance(value, float): + return lib.xl_write_float64(handle, value) + if isinstance(value, datetime.datetime): + if value.tzinfo is not None: + value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None) + delta = value - _EPOCH_DATETIME + return lib.xl_write_timestamp(handle, (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds) + if isinstance(value, datetime.date): + return lib.xl_write_date(handle, (value - _EPOCH_DATE).days) + if isinstance(value, datetime.time): + return lib.xl_write_time( + handle, (value.hour * 3600 + value.minute * 60 + value.second) * 1_000_000 + value.microsecond + ) + pointer, length = _encode(value) + return lib.xl_write_string(handle, pointer, length) def _finished_bytes(self, handle: ctypes.c_void_p) -> bytes: buffer = _native.NativeBuffer() diff --git a/python/tests/test_reader.py b/python/tests/test_reader.py index c5e2741e..cff06e66 100644 --- a/python/tests/test_reader.py +++ b/python/tests/test_reader.py @@ -16,12 +16,10 @@ OpenOptions, PasswordIncorrectError, PasswordRequiredError, - decode_cell, open_workbook, ) from excelreader import _native from excelreader import arrow_stream as arrow_stream_module -from excelreader import reader as reader_module ENCRYPTED = Path(__file__).resolve().parents[2] / "tests" / "ExcelReader.Tests" / "data" / "encrypted" @@ -85,26 +83,18 @@ def test_reads_from_bytes(xlsx_path): def test_exposes_sheet_metadata(xlsx_path): with open_workbook(xlsx_path) as workbook: - assert workbook.sheet_count >= 1 + assert len(workbook.sheets) >= 1 assert workbook.sheets[0].name assert workbook.is_date1904 is False -def test_sheet_names_matches_sheet_count_and_the_sheets(xlsx_path): - with open_workbook(xlsx_path) as workbook: - names = workbook.sheet_names - assert len(names) == workbook.sheet_count == len(workbook.sheets) - assert names[0] == workbook.sheets[0].name - assert [(sheet.index, sheet.name) for sheet in workbook.sheets] == list(enumerate(names)) - - -def test_sheet_names_does_not_disturb_row_enumeration(xlsx_path): +def test_listing_sheet_names_does_not_disturb_row_enumeration(xlsx_path): with open_workbook(xlsx_path) as workbook: rows = workbook.sheets[0].rows() first = next(rows) second = next(rows) - assert workbook.sheet_names + assert [sheet.name for sheet in workbook.sheets] third = next(rows) @@ -122,7 +112,7 @@ def test_use_after_close_raises(xlsx_path): workbook.close() with pytest.raises(ExcelReaderError): - workbook.sheet_count # noqa: B018 + len(workbook.sheets) def test_close_is_idempotent(xlsx_path): @@ -164,78 +154,12 @@ def test_as_date_returns_none_for_non_date_cells(xlsx_path): assert first_data_row[0].as_date() is None -def test_read_all_matches_row_by_row_iteration(xlsx_path): - with open_workbook(xlsx_path) as workbook: - all_rows = list(workbook.sheets[0].rows()) - - with open_workbook(xlsx_path) as workbook: - bulk_rows = workbook.sheets[0].read_all() - - assert bulk_rows == all_rows - - -def test_read_all_reads_the_whole_sheet_every_time(xlsx_path): +def test_rows_reads_the_whole_sheet_every_time(xlsx_path): with open_workbook(xlsx_path) as workbook: sheet = workbook.sheets[0] - first = sheet.read_all() + first = list(sheet.rows()) assert first - assert sheet.read_all() == first - - -def _columnar_to_rows(sheet) -> list[list[Cell]]: - rows = [] - for row_index in range(len(sheet.row_offsets) - 1): - start, end = sheet.row_offsets[row_index], sheet.row_offsets[row_index + 1] - rows.append([decode_cell(sheet, cell_index) for cell_index in range(start, end)]) - return rows - - -@pytest.mark.parametrize("fixture_name", ["xlsx_path", "xlsb_path"]) -def test_read_all_columnar_matches_read_all(fixture_name, request): - path = request.getfixturevalue(fixture_name) - - with open_workbook(path) as workbook: - expected = workbook.sheets[0].read_all() - - with open_workbook(path) as workbook: - sheet = workbook.sheets[0].read_all_columnar() - - assert _columnar_to_rows(sheet) == expected - - -def test_read_all_columnar_without_numpy(xlsx_path, monkeypatch): - monkeypatch.setattr(reader_module, "_numpy", None) - - with open_workbook(xlsx_path) as workbook: - sheet = workbook.sheets[0].read_all_columnar() - - from array import array - - assert isinstance(sheet.columns, array) - with open_workbook(xlsx_path) as workbook: - assert _columnar_to_rows(sheet) == workbook.sheets[0].read_all() - - -def test_read_all_columnar_grows_the_buffer(xlsx_path, monkeypatch): - monkeypatch.setattr(reader_module, "_INITIAL_ALL_ROWS_BUFFER", 4) - - with open_workbook(xlsx_path) as workbook: - grown = workbook.sheets[0].read_all_columnar() - - with open_workbook(xlsx_path) as workbook: - expected = workbook.sheets[0].read_all() - - assert _columnar_to_rows(grown) == expected - - -def test_read_all_columnar_reads_the_whole_sheet_every_time(xlsx_path): - with open_workbook(xlsx_path) as workbook: - sheet = workbook.sheets[0] - first = sheet.read_all_columnar() - assert len(first.columns) > 0 - assert _columnar_to_rows(sheet.read_all_columnar()) == _columnar_to_rows(first) - - + assert list(sheet.rows()) == first @pytest.fixture @@ -343,15 +267,14 @@ def test_parse_typed_reads_the_whole_sheet_regardless_of_the_row_cursor(typed_cs assert len(rest) == 2 -def test_to_arrow_returns_a_struct_array_matching_parse_typed(typed_csv): +def test_to_arrow_returns_a_record_batch_matching_parse_typed(typed_csv): pa = pytest.importorskip("pyarrow") with open_workbook(typed_csv) as workbook: - array = workbook.sheets[0].to_arrow(_TYPED_SCHEMA) + batch = workbook.sheets[0].to_arrow(_TYPED_SCHEMA) - assert isinstance(array, pa.StructArray) - assert len(array) == 2 - batch = pa.RecordBatch.from_struct_array(array) + assert isinstance(batch, pa.RecordBatch) + assert batch.num_rows == 2 assert batch.schema.names == ["name", "qty", "price", "flag", "day", "clock", "stamp"] assert batch.column("name").to_pylist() == ["widget", "gadget"] assert batch.column("qty").to_pylist() == [3, 7] @@ -364,10 +287,10 @@ def test_to_arrow_survives_the_workbook_being_closed(typed_csv): pytest.importorskip("pyarrow") with open_workbook(typed_csv) as workbook: - array = workbook.sheets[0].to_arrow([ColumnSpec(ColumnType.I64, name="qty")]) + batch = workbook.sheets[0].to_arrow([ColumnSpec(ColumnType.I64, name="qty")]) gc.collect() - assert array.field(0).to_pylist() == [3, 7] + assert batch.column(0).to_pylist() == [3, 7] def test_to_arrow_raises_for_an_unknown_column_name(typed_csv): @@ -377,17 +300,6 @@ def test_to_arrow_raises_for_an_unknown_column_name(typed_csv): workbook.sheets[0].to_arrow([ColumnSpec(ColumnType.I64, name="nope")]) -def test_to_record_batch_returns_a_column_named_batch_matching_parse_typed(typed_csv): - pa = pytest.importorskip("pyarrow") - - with open_workbook(typed_csv) as workbook: - batch = workbook.sheets[0].to_record_batch(_TYPED_SCHEMA) - - assert isinstance(batch, pa.RecordBatch) - assert batch.schema.names == ["name", "qty", "price", "flag", "day", "clock", "stamp"] - assert batch.column("qty").to_pylist() == [3, 7] - - def test_to_pandas_returns_a_dataframe_matching_parse_typed(typed_csv): pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") @@ -455,15 +367,13 @@ def test_open_options_limit_actually_aborts_an_oversized_read(tmp_path): with open_workbook(path, format="csv") as workbook: assert len(list(workbook.sheets[0].rows())[1][0].value) == 200_000 - with pytest.raises(ExcelReaderError), open_workbook(path, format="csv", options=OpenOptions(csv_max_cell_bytes=100_000)) as workbook: + with pytest.raises(ExcelReaderError), open_workbook(path, format="csv", options=OpenOptions(max_cell_bytes=100_000)) as workbook: list(workbook.sheets[0].rows()) - - def test_opens_encrypted_workbook_with_password(): with open_workbook(ENCRYPTED / "agile-aes256-sha512.xlsx", password="hunter2") as book: - assert book.sheet_count > 0 + assert len(book.sheets) > 0 def test_reports_password_required_without_password(): @@ -478,9 +388,9 @@ def test_reports_password_incorrect_with_wrong_password(): def test_encrypted_rows_match_plaintext(): with open_workbook(ENCRYPTED / "agile-aes256-sha512.xlsx", password="hunter2") as enc: - encrypted_rows = enc.sheets[0].read_all() + encrypted_rows = list(enc.sheets[0].rows()) with open_workbook(ENCRYPTED / "agile-aes256-sha512.plain.xlsx") as plain: - assert plain.sheets[0].read_all() == encrypted_rows + assert list(plain.sheets[0].rows()) == encrypted_rows def test_password_is_not_in_repr(): @@ -575,28 +485,21 @@ def test_abandoning_the_generator_closes_the_reader(batched_csv, monkeypatch): -def test_to_record_batch_reader_matches_to_record_batch(batched_csv): - pytest.importorskip("pyarrow") +def test_arrow_stream_read_by_pyarrow_matches_to_arrow(batched_csv): + pa = pytest.importorskip("pyarrow") with open_workbook(batched_csv) as workbook: - expected = workbook.sheets[0].to_record_batch(_BATCH_SCHEMA) + expected = workbook.sheets[0].to_arrow(_BATCH_SCHEMA) with open_workbook(batched_csv) as workbook: - reader = workbook.sheets[0].to_record_batch_reader(_BATCH_SCHEMA, batch_size=8) + reader = pa.RecordBatchReader.from_stream(workbook.sheets[0].to_arrow_stream(_BATCH_SCHEMA, batch_size=8)) assert reader.schema == expected.schema batches = list(reader) - assert sum(batch.num_rows for batch in batches) == expected.num_rows + assert pa.Table.from_batches(batches).equals(pa.Table.from_batches([expected])) assert len(batches) == 7, "50 rows at batch size 8 is 7 batches" assert all(batch.num_rows <= 8 for batch in batches) -def test_to_record_batch_reader_rejects_a_negative_batch_size(batched_csv): - pytest.importorskip("pyarrow") - with open_workbook(batched_csv) as workbook: - with pytest.raises(ExcelReaderError): - workbook.sheets[0].to_record_batch_reader(_BATCH_SCHEMA, batch_size=-1) - - def test_arrow_stream_builds_a_polars_dataframe_in_batches(batched_csv): pl = pytest.importorskip("polars") with open_workbook(batched_csv) as workbook: @@ -705,46 +608,28 @@ def test_arrow_stream_rejects_a_negative_batch_size(batched_csv): def test_a_stream_and_a_reader_on_one_sheet_each_read_every_row(batched_csv): - pytest.importorskip("pyarrow") + pa = pytest.importorskip("pyarrow") with open_workbook(batched_csv) as workbook: sheet = workbook.sheets[0] reader = sheet.iter_parse_typed(_BATCH_SCHEMA, batch_size=4) next(reader) - stream = sheet.to_record_batch_reader(_BATCH_SCHEMA, batch_size=4) + stream = pa.RecordBatchReader.from_stream(sheet.to_arrow_stream(_BATCH_SCHEMA, batch_size=4)) assert sum(batch.row_count for batch in reader) == 46 assert sum(batch.num_rows for batch in stream) == 50 def test_a_whole_sheet_read_leaves_an_open_stream_alone(batched_csv): - pytest.importorskip("pyarrow") + pa = pytest.importorskip("pyarrow") with open_workbook(batched_csv) as workbook: sheet = workbook.sheets[0] - reader = sheet.to_record_batch_reader(_BATCH_SCHEMA, batch_size=4) + reader = pa.RecordBatchReader.from_stream(sheet.to_arrow_stream(_BATCH_SCHEMA, batch_size=4)) reader.read_next_batch() assert sheet.parse_typed(_BATCH_SCHEMA).row_count == 50 assert sum(batch.num_rows for batch in reader) == 46 -def test_iter_pandas_yields_one_frame_per_batch(batched_csv): - pytest.importorskip("pyarrow") - pytest.importorskip("pandas") - with open_workbook(batched_csv) as workbook: - frames = list(workbook.sheets[0].iter_pandas(_BATCH_SCHEMA, batch_size=8)) - assert len(frames) == 7 - assert sum(len(frame) for frame in frames) == 50 - - -def test_iter_polars_yields_one_frame_per_batch(batched_csv): - pytest.importorskip("pyarrow") - pytest.importorskip("polars") - with open_workbook(batched_csv) as workbook: - frames = list(workbook.sheets[0].iter_polars(_BATCH_SCHEMA, batch_size=8)) - assert len(frames) == 7 - assert sum(frame.height for frame in frames) == 50 - - def test_to_polars_stays_chunked(batched_csv): polars = pytest.importorskip("polars") with open_workbook(batched_csv) as workbook: @@ -802,6 +687,16 @@ def test_to_polars_in_parallel_matches_the_streamed_frame(): assert parallel.equals(streamed) +def test_to_pandas_in_parallel_matches_the_streamed_frame(): + pytest.importorskip("pyarrow") + pytest.importorskip("pandas") + with open_workbook(_FIXTURE_CSV) as workbook: + streamed = workbook.sheets[0].to_pandas(_FIXTURE_SCHEMA) + parallel = workbook.sheets[0].to_pandas(_FIXTURE_SCHEMA, parallelism=0) + + assert parallel.equals(streamed) + + def test_parse_typed_rejects_a_negative_parallelism(typed_csv): with open_workbook(typed_csv) as workbook, pytest.raises(ExcelReaderError): workbook.sheets[0].parse_typed(_TYPED_SCHEMA, parallelism=-1) diff --git a/python/tests/test_sheets.py b/python/tests/test_sheets.py index 4eeb3789..db865117 100644 --- a/python/tests/test_sheets.py +++ b/python/tests/test_sheets.py @@ -117,15 +117,13 @@ def test_rows_opened_before_close_read_to_the_end_after_it(): assert [row[0].value for row in rows] == ["b2", "b3"] -def test_read_all_and_parse_typed_read_the_named_sheet(): +def test_typed_reads_read_the_named_sheet(): from excelreader import ColumnSpec, ColumnType with open_workbook(_two_sheets()) as workbook: second = workbook.sheets[1] schema = [ColumnSpec(ColumnType.STRING, index=0, nullable=True)] - assert len(second.read_all()) == 3 - assert len(second.read_all_columnar().row_offsets) == 4 assert second.parse_typed(schema, header_row=0).row_count == 3 assert sum(table.row_count for table in second.iter_parse_typed(schema, header_row=0, batch_size=2)) == 3 assert len(second.infer_schema(header_row=0)) == 1 @@ -157,15 +155,15 @@ def test_typed_parses_of_every_sheet_run_on_separate_threads(): assert int(table.columns[0][0]) == sheet * 1_000_000 -def test_a_record_batch_reader_outlives_its_workbook(): +def test_an_arrow_stream_outlives_its_workbook(): pyarrow = pytest.importorskip("pyarrow") from excelreader import ColumnSpec, ColumnType schema = [ColumnSpec(ColumnType.I64, index=1)] with open_workbook(_many_sheets()) as workbook: - reader = workbook.sheets[2].to_record_batch_reader(schema, header_row=0, batch_size=100) + stream = workbook.sheets[2].to_arrow_stream(schema, header_row=0, batch_size=100) - table = reader.read_all() + table = pyarrow.RecordBatchReader.from_stream(stream).read_all() assert isinstance(table, pyarrow.Table) assert table.num_rows == _PARALLEL_ROWS @@ -192,12 +190,7 @@ def test_a_sheet_with_no_rows_reads_empty(): data = buffer.getvalue() with open_workbook(data) as workbook: - sheet = workbook.sheets[0] - assert sheet.read_all() == [] - columnar = sheet.read_all_columnar() - assert list(columnar.row_offsets) == [0] - assert columnar.values == b"" - assert list(sheet.rows()) == [] + assert list(workbook.sheets[0].rows()) == [] class _SlowCheckWorkbook(Workbook): diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index d911b7d0..000893bb 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -9,9 +9,6 @@ WriteOptions, encrypt_package, open_workbook, - write_arrow, - write_pandas, - write_polars, write_workbook, ) @@ -71,7 +68,7 @@ def test_write_workbook_applies_the_sheet_name(source_csv, tmp_path): write_workbook(out, table, _TYPES, options=WriteOptions(sheet_name="Vendas")) with open_workbook(out) as workbook: - assert workbook.sheet_names == ["Vendas"] + assert [sheet.name for sheet in workbook.sheets] == ["Vendas"] def test_write_workbook_rejects_an_unknown_extension(source_csv, tmp_path): @@ -90,12 +87,37 @@ def test_write_workbook_reports_a_native_rejection(source_csv, tmp_path): write_workbook(tmp_path / "out.xlsx", table, _TYPES, options=WriteOptions(sheet_name="has/slash")) -def test_write_arrow_round_trips(tmp_path): +def test_write_workbook_rejects_a_typed_table_without_types(source_csv, tmp_path): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse_typed(_SCHEMA) + + with pytest.raises(TypeError, match="types"): + write_workbook(tmp_path / "out.xlsx", table) + + +def test_write_workbook_rejects_types_for_arrow_data(tmp_path): pa = pytest.importorskip("pyarrow") - batch = pa.RecordBatch.from_pydict({"name": ["widget", "gadget"], "qty": [3, 7]}) + batch = pa.RecordBatch.from_pydict({"name": ["widget"], "qty": [3]}) + with pytest.raises(TypeError, match="types"): + write_workbook(tmp_path / "out.xlsx", batch, _TYPES) + + +def test_write_workbook_rejects_data_it_cannot_write(tmp_path): + pytest.importorskip("pyarrow") + + with pytest.raises(TypeError, match="data"): + write_workbook(tmp_path / "out.xlsx", [["widget", 3]]) + + +@pytest.mark.parametrize("kind", ["record_batch", "table"]) +def test_write_workbook_writes_arrow_data(tmp_path, kind): + pa = pytest.importorskip("pyarrow") + + data = {"name": ["widget", "gadget"], "qty": [3, 7]} + frame = pa.RecordBatch.from_pydict(data) if kind == "record_batch" else pa.table(data) out = tmp_path / "arrow.xlsx" - write_arrow(out, batch) + write_workbook(out, frame) with open_workbook(out) as workbook: result = workbook.sheets[0].parse_typed(_SCHEMA) @@ -104,12 +126,28 @@ def test_write_arrow_round_trips(tmp_path): assert list(result.columns[1]) == [3, 7] -def test_write_pandas_round_trips(tmp_path): +def test_write_workbook_leaves_a_pandas_index_out(tmp_path): pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") out = tmp_path / "pandas.xlsx" - write_pandas(out, pd.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]})) + write_workbook(out, pd.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]}, index=[10, 20])) + + with open_workbook(out) as workbook: + rows = [[cell.value for cell in row] for row in workbook.sheets[0].rows()] + + assert rows == [["name", "qty"], ["widget", "3"], ["gadget", "7"]] + + +def test_write_workbook_writes_any_arrow_c_stream(tmp_path): + pa = pytest.importorskip("pyarrow") + + class Producer: + def __arrow_c_stream__(self, requested_schema=None): + return pa.table({"name": ["widget", "gadget"], "qty": [3, 7]}).__arrow_c_stream__(requested_schema) + + out = tmp_path / "stream.xlsx" + write_workbook(out, Producer()) with open_workbook(out) as workbook: result = workbook.sheets[0].parse_typed(_SCHEMA) @@ -118,7 +156,7 @@ def test_write_pandas_round_trips(tmp_path): assert list(result.columns[1]) == [3, 7] -def test_write_pandas_round_trips_a_multi_chunk_frame(tmp_path): +def test_write_workbook_round_trips_a_multi_chunk_pandas_frame(tmp_path): pa = pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") @@ -132,7 +170,7 @@ def test_write_pandas_round_trips_a_multi_chunk_frame(tmp_path): assert pa.Table.from_pandas(frame, preserve_index=False).column(0).num_chunks > 1 out = tmp_path / "chunked.xlsx" - write_pandas(out, frame) + write_workbook(out, frame) with open_workbook(out) as workbook: result = workbook.sheets[0].parse_typed(_SCHEMA) @@ -141,29 +179,29 @@ def test_write_pandas_round_trips_a_multi_chunk_frame(tmp_path): assert list(result.columns[1]) == [3, 7] -def test_write_pandas_writes_a_header_only_sheet_for_an_empty_frame(tmp_path): +def test_write_workbook_writes_a_header_only_sheet_for_an_empty_frame(tmp_path): pytest.importorskip("pyarrow") pd = pytest.importorskip("pandas") out = tmp_path / "empty.xlsx" - write_pandas( + write_workbook( out, pd.DataFrame({"name": pd.Series([], dtype="string"), "qty": pd.Series([], dtype="int64")}), ) with open_workbook(out) as workbook: - rows = workbook.sheets[0].read_all() + rows = list(workbook.sheets[0].rows()) assert [cell.value for cell in rows[0]] == ["name", "qty"] assert len(rows) == 1 -def test_write_polars_round_trips(tmp_path): +def test_write_workbook_writes_a_polars_frame(tmp_path): pytest.importorskip("pyarrow") pl = pytest.importorskip("polars") out = tmp_path / "polars.xlsx" - write_polars(out, pl.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]})) + write_workbook(out, pl.DataFrame({"name": ["widget", "gadget"], "qty": [3, 7]})) with open_workbook(out) as workbook: result = workbook.sheets[0].parse_typed(_SCHEMA) @@ -172,13 +210,13 @@ def test_write_polars_round_trips(tmp_path): assert list(result.columns[1]) == [3, 7] -def test_write_arrow_rejects_an_unsupported_arrow_type(tmp_path): +def test_write_workbook_rejects_an_unsupported_arrow_type(tmp_path): pa = pytest.importorskip("pyarrow") batch = pa.RecordBatch.from_pydict({"nested": [[1, 2], [3]]}) with pytest.raises(ValueError, match="type"): - write_arrow(tmp_path / "bad.xlsx", batch) + write_workbook(tmp_path / "bad.xlsx", batch) def test_encrypt_package_round_trips_through_open_workbook(source_csv, tmp_path): @@ -234,11 +272,8 @@ def _to_bytes(table): return buffer.getvalue() -@pytest.mark.parametrize( - ("writer", "frame_module"), - [(write_arrow, None), (write_pandas, "pandas"), (write_polars, "polars")], -) -def test_frame_writers_round_trip_through_a_bytesio(writer, frame_module): +@pytest.mark.parametrize("frame_module", [None, "pandas", "polars"]) +def test_frames_round_trip_through_a_bytesio(frame_module): pa = pytest.importorskip("pyarrow") data = {"name": ["widget", "gadget"], "qty": [3, 7]} if frame_module is None: @@ -247,7 +282,7 @@ def test_frame_writers_round_trip_through_a_bytesio(writer, frame_module): frame = pytest.importorskip(frame_module).DataFrame(data) buffer = io.BytesIO() - writer(buffer, frame, format="xlsx", options=WriteOptions(sheet_name="Vendas")) + write_workbook(buffer, frame, format="xlsx", options=WriteOptions(sheet_name="Vendas")) result = _parse_bytes(buffer.getvalue(), "xlsx") assert list(result.columns[0]) == ["widget", "gadget"] diff --git a/python/tests/test_writer_handle.py b/python/tests/test_writer_handle.py index db82d7fe..0f7f2f6a 100644 --- a/python/tests/test_writer_handle.py +++ b/python/tests/test_writer_handle.py @@ -4,21 +4,15 @@ import pytest -from excelreader import CellType, ColumnType, open_workbook, open_writer +from excelreader import CellType, ColumnSpec, ColumnType, open_workbook, open_writer def test_round_trips_typed_cells(tmp_path): path = tmp_path / "out.xlsx" with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str("name") - writer.write_str("qty") - writer.end_row() - writer.start_row() - writer.write_str("widget") - writer.write_i64(7) - writer.end_row() + writer.write_row(["name", "qty"]) + writer.write_row(["widget", 7]) writer.end_sheet() with open_workbook(path) as workbook: @@ -33,7 +27,7 @@ def test_round_trips_typed_cells(tmp_path): strict=True, reason=( "native reader bug: XlsxReader.Enumerator.cs ParseCellSpan skips self-closing elements " - "instead of recording them as blank cells, so a trailing blank cell written by write_null is " + "instead of recording them as blank cells, so a trailing typed blank cell is " "missing from the decoded row (len 7, not 8). Pre-existing in the native core, out of scope " "for Task 6 (Python bindings only) - remove this marker once the native reader is fixed." ), @@ -42,16 +36,18 @@ def test_writes_every_scalar_type(tmp_path): path = tmp_path / "types.xlsx" with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str("s") - writer.write_i64(1) - writer.write_f64(1.5) - writer.write_bool(True) - writer.write_date(datetime.date(2020, 3, 4)) - writer.write_time(datetime.time(13, 45, 6)) - writer.write_timestamp(datetime.datetime(2020, 3, 4, 13, 45, 6)) - writer.write_null(ColumnType.I64) - writer.end_row() + writer.write_row( + [ + "s", + 1, + 1.5, + True, + datetime.date(2020, 3, 4), + datetime.time(13, 45, 6), + datetime.datetime(2020, 3, 4, 13, 45, 6), + (None, ColumnType.I64), + ] + ) writer.end_sheet() with open_workbook(path) as workbook: @@ -62,14 +58,11 @@ def test_writes_every_scalar_type(tmp_path): assert len(rows[0]) == 8 -def test_write_str_none_is_a_blank_cell(tmp_path): +def test_none_is_a_blank_cell(tmp_path): path = tmp_path / "blank.xlsx" with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str(None) - writer.write_str("after") - writer.end_row() + writer.write_row([None, "after"]) writer.end_sheet() with open_workbook(path) as workbook: @@ -78,16 +71,42 @@ def test_write_str_none_is_a_blank_cell(tmp_path): assert rows[0][-1] == "after" +def test_a_typed_blank_reads_back_as_a_null_of_that_type(tmp_path): + path = tmp_path / "typed-blank.xlsx" + with open_writer(path) as writer: + writer.start_sheet("Data") + writer.write_row(["qty"]) + writer.write_row([(None, ColumnType.I64)]) + writer.write_row([5]) + writer.end_sheet() + + with open_workbook(path) as workbook: + assert workbook.sheets[0].infer_schema() == [ColumnSpec(ColumnType.I64, name="qty", nullable=True)] + table = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.I64, name="qty", nullable=True)]) + + assert table.row_count == 2 + assert table.validity[0][0] & 0b11 == 0b10 + assert int(table.columns[0][1]) == 5 + + +@pytest.mark.parametrize("value", [(1, ColumnType.I64), (None, 1), (None,), [None, ColumnType.I64]]) +def test_write_row_rejects_a_malformed_typed_blank(tmp_path, value): + with open_writer(tmp_path / "bad-blank.xlsx") as writer: + writer.start_sheet("Data") + with pytest.raises(TypeError): + writer.write_row(["fine", value]) + writer.end_sheet() + + def test_out_of_order_call_raises(tmp_path): from excelreader import ExcelReaderError path = tmp_path / "bad.xlsx" with open_writer(path) as writer: with pytest.raises(ExcelReaderError): - writer.write_str("too early") + writer.write_row(["too early"]) writer.start_sheet("Data") - writer.start_row() - writer.end_row() + writer.write_row([]) writer.end_sheet() @@ -95,9 +114,7 @@ def test_closing_twice_is_safe(tmp_path): path = tmp_path / "twice.xlsx" writer = open_writer(path) writer.start_sheet("Data") - writer.start_row() - writer.write_i64(1) - writer.end_row() + writer.write_row([1]) writer.end_sheet() writer.close() writer.close() @@ -142,30 +159,20 @@ def test_write_row_rejects_an_unsupported_type(tmp_path): assert "1" in str(excinfo.value), "the message names the position" -def test_write_row_matches_explicit_calls(tmp_path): - sugar = tmp_path / "sugar.xlsx" - explicit = tmp_path / "explicit.xlsx" - - with open_writer(sugar) as writer: - writer.start_sheet("Data") - writer.write_row(["a", 1, 2.5, False]) - writer.end_sheet() - - with open_writer(explicit) as writer: +def test_an_aware_datetime_is_written_in_utc(tmp_path): + path = tmp_path / "aware.xlsx" + offset = datetime.timezone(datetime.timedelta(hours=-3)) + with open_writer(path) as writer: writer.start_sheet("Data") - writer.start_row() - writer.write_str("a") - writer.write_i64(1) - writer.write_f64(2.5) - writer.write_bool(False) - writer.end_row() + writer.write_row(["at"]) + writer.write_row([datetime.datetime(2021, 6, 7, 5, 0, tzinfo=offset)]) + writer.write_row([datetime.datetime(2021, 6, 7, 8, 0)]) writer.end_sheet() - def read(path): - with open_workbook(path) as workbook: - return [[(cell.type, cell.value) for cell in row] for row in workbook.sheets[0].rows()] + with open_workbook(path) as workbook: + stamps = workbook.sheets[0].parse_typed([ColumnSpec(ColumnType.TIMESTAMP, name="at")]).columns[0] - assert read(sugar) == read(explicit) + assert stamps[0] == stamps[1] def test_a_writer_over_a_bytesio_hands_it_the_workbook_on_close(): @@ -245,10 +252,5 @@ def test_an_integer_beyond_64_bits_is_refused_not_wrapped(tmp_path, value): writer.start_sheet("Data") with pytest.raises(OverflowError): writer.write_row(["fine", value]) - writer.start_row() - with pytest.raises(OverflowError): - writer.write_i64(value) - writer.write_i64(2**63 - 1) - writer.write_i64(-(2**63)) - writer.end_row() + writer.write_row([2**63 - 1, -(2**63)]) writer.end_sheet() From 3d805d1ff3032024ae9a3c04172dba2654598041 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 16:51:14 -0300 Subject: [PATCH 08/21] docs: list the Python API reductions in the v7 migration guide Co-Authored-By: Claude Opus 5.5 --- docs/guide/migrating-to-v7.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md index 9b8df738..73059d4c 100644 --- a/docs/guide/migrating-to-v7.md +++ b/docs/guide/migrating-to-v7.md @@ -185,3 +185,24 @@ The streaming writer and the encryption helper follow the same pattern. `open_wr |---|---| | `with open_writer_to_memory("xlsx") as w:`
` ...`
` payload = w.bytes()` | `buffer = io.BytesIO()`
`with open_writer(buffer, "xlsx") as w:`
` ...`
`payload = buffer.getvalue()` | | `secret = encrypt_package_bytes(payload, password)` | `encrypt_package(payload, out, password)` with `out = io.BytesIO()` | + +### Python: one shape per read + +A sheet reads as rows (`rows()`), a typed table (`parse_typed()`, or `iter_parse_typed()` in batches), Arrow (`to_arrow()`, or `to_arrow_stream()` in batches) or a DataFrame (`to_pandas()`, `to_polars()`). `write_workbook()` takes a `TypedTable` or any Arrow-compatible table, including pandas and polars DataFrames. `SheetWriter` writes whole rows only. + +| v6 | v7 | +|---|---| +| `workbook.sheet_count` | `len(workbook.sheets)` | +| `workbook.sheet_names` | `[s.name for s in workbook.sheets]` | +| `workbook.sheet_name_at(i)` | `workbook.sheets[i].name` | +| `sheet.read_all()` | `list(sheet.rows())`, or `parse_typed()` for speed | +| `sheet.read_all_columnar()`, `ColumnarSheet`, `decode_cell()` | `sheet.parse_typed(sheet.infer_schema())` | +| `sheet.to_arrow()` returning a `StructArray` | `sheet.to_arrow()` returns a `pyarrow.RecordBatch` | +| `sheet.to_record_batch(...)` | `sheet.to_arrow(...)` | +| `sheet.to_record_batch_reader(...)` | `pyarrow.RecordBatchReader.from_stream(sheet.to_arrow_stream(...))` | +| `sheet.iter_pandas(...)`, `sheet.iter_polars(...)` | iterate the reader above and convert each batch | +| `write_arrow(dest, batch)`, `write_pandas(dest, df)`, `write_polars(dest, df)` | `write_workbook(dest, batch_or_df)` | +| `writer.start_row()`, `write_str(v)` … `write_timestamp(v)`, `end_row()` | `writer.write_row([...])` | +| `writer.write_null(ColumnType.X)` | `(None, ColumnType.X)` as a value in `write_row` | +| `OpenOptions(csv_max_cell_bytes=n)`, `OpenOptions(csv_intern_strings=b)` | `OpenOptions(max_cell_bytes=n)`, `OpenOptions(intern_strings=b)`, which now apply to CSV too | +| `from excelreader import ArrowStream` | `from excelreader.arrow_stream import ArrowStream` | From 0f20dfe019901371fa9b209208a6269ad897073f Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 17:17:52 -0300 Subject: [PATCH 09/21] feat(cpp)!: slim the C++ binding's public API for 7.0 One way to do each thing, with options passed as `const Opts& = {}`: - Workbook::open is one overload set (path, byte span, unique_ptr, unique_ptr); open_memory, open_source and open_stream are gone. The format moves into OpenOptions::format. - Workbook::sheet_count, sheet_name_at and sheet_names are removed; use sheets(), sheet(i) and Sheet::name(). - Sheet::read_all_decoded is renamed read_all; RowCursor::read_all_decoded is removed. typed_reader and arrow_stream reject batch_size <= 0. - write_columns and write_sheet take a path or a std::vector& and read the format from WriteOptions::format (AUTO = path extension). write_columns_to_memory, write_sheet_to_memory and the format overloads are removed; format_from_path is internal. - WriterHandle::open takes a path or a vector; open_memory and bytes() are removed and finish() reports the result of completing the workbook. - aggregate_csv(path | span, seed, OpenOptions, header_row, dop) replaces the four aggregate_csv_file/aggregate_csv_memory functions and CsvParallelOptions; the seed is a value copied per partition. - The C ABI is hidden: raw-handle constructors, from_raw and handle() are private or removed, to_c() is internal, XlType and FieldBinding move to detail, TableView::to_vector is removed. - ArrowSchemaGuard and ArrowArrayGuard merge into ArrowTable; every stream batch carries its schema. - OpenOptions::password_ and password() become a plain `password` field, and csv_max_cell_bytes/csv_intern_strings are dropped: max_cell_bytes and intern_strings now apply to CSV too. - RowView's unused payload decoding is removed. BREAKING CHANGE: the C++ header API changes as listed above; see the 7.0 migration guide for the mapping. Co-Authored-By: Claude Opus 5.5 --- cpp/README.md | 121 +- cpp/benchmarks/benchmark_compare.cpp | 4 +- cpp/benchmarks/benchmark_csv_compare.cpp | 4 +- cpp/benchmarks/benchmark_workbook.cpp | 16 +- cpp/benchmarks/benchmark_write.cpp | 6 +- cpp/benchmarks/benchmark_write_compare.cpp | 8 +- cpp/tests/aggregate.cpp | 113 +- cpp/tests/arrow.cpp | 47 +- cpp/tests/batch.cpp | 27 +- cpp/tests/package/main.cpp | 2 +- cpp/tests/rows.cpp | 20 +- cpp/tests/sheets.cpp | 39 +- cpp/tests/smoke.cpp | 45 +- cpp/tests/sources.cpp | 69 +- cpp/tests/write.cpp | 157 +- .../include/excelreader.hpp | 1322 +++++++---------- .../include/excelreader_arrow.hpp | 143 +- 17 files changed, 928 insertions(+), 1215 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index d756c881..c3324584 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -2,8 +2,8 @@ # excelreader (C++) -Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path or -memory, with the full open-options surface), sheet navigation, schema inference, schema-driven +Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path, +memory, a random-access `xl::Source` or an `xl::InputStream`, with the full open-options surface), sheet navigation, schema inference, schema-driven typed table parsing, schema-driven writing, row-by-row decoded reads, and Arrow export. ## Requirements @@ -20,7 +20,7 @@ typed table parsing, schema-driven writing, row-by-row decoded reads, and Arrow include(FetchContent) FetchContent_Declare(excelreader GIT_REPOSITORY https://github.com/GabrielMarquezMatte/ExcelReader.git - GIT_TAG v4.0.1 + GIT_TAG v7.0.0 SOURCE_SUBDIR cpp) FetchContent_MakeAvailable(excelreader) @@ -47,14 +47,14 @@ cmake --install build/cpp --config Release --prefix /your/prefix ``` ```cmake -find_package(excelreader 3.0 REQUIRED) +find_package(excelreader 7.0 REQUIRED) target_link_libraries(your_app PRIVATE xl::excelreader) ``` Point `CMAKE_PREFIX_PATH` at the prefix you installed into. The native binary is resolved once, at install time — `find_package` never touches the network. The package version comes from `EXCELREADER_VERSION`, so a checkout that isn't on a tag installs as `0.0.0` and any versioned -`find_package` request against it fails; pass `-DEXCELREADER_VERSION=v3.0.2` to install under a real +`find_package` request against it fails; pass `-DEXCELREADER_VERSION=v7.0.0` to install under a real version. On Windows the generated import library is installed next to the DLL, and the package ships `excelreader_copy_native_library` as well, so a `find_package` consumer copies the DLL next to its executable the same way a `FetchContent` one does. @@ -100,7 +100,7 @@ for (const auto& row : *table) { /* ... */ } ### Sheets and schema inference ```cpp -for (const auto& name : *workbook->sheet_names()) { /* ... */ } +for (xl::Sheet sheet : *workbook->sheets()) { std::println("{}", sheet.name().value()); } // Guess a schema from the header row plus a sample of the data, before committing to one. for (const auto& column : *workbook->sheet(1).infer_schema(1, 100)) { @@ -171,11 +171,11 @@ while (auto row = cursor.next_row()) ``` Each row is invalidated by the next `next_row()`. To hold every row at once, use -`read_all_decoded()`, which decodes the whole sheet in one native call and owns it until -destroyed (`RowCursor::read_all_decoded()` does the same for the rows a cursor has left): +`Sheet::read_all()`, which decodes the whole sheet in one native call and owns it until +destroyed: ```cpp -auto rows = workbook.sheet(0).read_all_decoded().value(); +auto rows = workbook.sheet(0).read_all().value(); for (auto row : rows) { std::println("{} cells", row.size()); @@ -202,21 +202,19 @@ for (auto &batch : reader) `typed_reader` returns `std::expected, xl::Error>`; `next()` returns `std::expected>, Error>`, an empty optional at end of sheet, and `TypedReader` is also an input range, so the range-`for` above works directly. `batch_size` is rows -per batch — `0` means one unbounded batch, identical to `parse_sheet`; negative is an error. Each -batch is independent and outlives the reader. +per batch and must be positive (`0` or negative is `XL_INVALID_ARGUMENT`; read a whole sheet with +`parse_sheet`). Each batch is independent and outlives the reader. Several typed readers and Arrow streams may be open on one workbook at once; each read is independent. ### Encrypted workbooks -`OpenOptions::password(std::string_view)` unlocks a password-protected OOXML workbook -(.xlsx/.xlsb/.xlsm) through `Workbook::open`/`open_memory`: +`OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm) through +any `Workbook::open` overload: ```cpp -xl::OpenOptions options{}; -options.password("hunter2"); -auto workbook = xl::Workbook::open("protected.xlsx", XL_FORMAT_AUTO, &options); +auto workbook = xl::Workbook::open("protected.xlsx", {.password = "hunter2"}); if (!workbook) { if (workbook.error().code == XL_STATUS_PASSWORD_REQUIRED) { /* no password was supplied */ } else if (workbook.error().code == XL_STATUS_PASSWORD_INCORRECT) { /* ask again */ } @@ -226,26 +224,22 @@ if (!workbook) { Unlike the Rust/Python bindings, this header never throws — `XL_STATUS_PASSWORD_REQUIRED` and `XL_STATUS_PASSWORD_INCORRECT` (from `excelreader.h`) are reported as ordinary `xl::Error::code` -values on the `std::unexpected` returned from `open`/`open_memory`, distinguishable from any other -failure without parsing `error().message`. - -The `password_` string this builds into `xl_open_options::password` must outlive the open call; -keeping it in the `OpenOptions` object (which owns its own copy) is what makes -`options.password("hunter2")` safe to call with a temporary, as above. +values on the `std::unexpected` returned from `open`, distinguishable from any other failure without +parsing `error().message`. Writing an encrypted workbook is a second step: write the plaintext package with `write_columns`/ `write_sheet`, then wrap it with `xl::encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container — AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: ```cpp -auto written = xl::write_columns("plain.xlsx", XL_FORMAT_XLSX, columns); +auto written = xl::write_columns("plain.xlsx", columns); auto encrypted = xl::encrypt_package("plain.xlsx", "secret.xlsx", "hunter2"); if (!encrypted) { /* encrypted.error().message */ } ``` `package_path` is read twice (it is not disposed or removed), so it must already be a finished file. Encryption parameters are fixed at Excel's own defaults — there are no options — and only XLSX/XLSB -packages can be encrypted, matching what `Workbook::open`/`open_memory` can decrypt. +packages can be encrypted, matching what `Workbook::open` can decrypt. ### Reading from your own bytes @@ -268,7 +262,7 @@ public: } }; -auto workbook = xl::Workbook::open_stream(std::make_unique(), XL_FORMAT_CSV); +auto workbook = xl::Workbook::open(std::make_unique(), {.format = XL_FORMAT_CSV}); ``` The workbook owns the object you pass and deletes it once nothing needs it. A returned error @@ -289,9 +283,9 @@ auto table = xl::parse_arrow(workbook->sheet(0)); // table->array / table->schema are a top-level struct array; both release in ~ArrowTable. ``` -Batched: `xl::arrow_stream` is `xl::parse_arrow` delivered a batch at a time. `schema()` and -`next()` return RAII guards (`ArrowSchemaGuard`, `ArrowArrayGuard`) that release exactly once, and -each batch outlives the stream that produced it: +Batched: `xl::arrow_stream` is `xl::parse_arrow` delivered a batch at a time. `next()` returns +an `xl::ArrowTable` holding the batch and its own copy of the schema (`schema()` returns one whose +`array` is empty); each releases exactly once and outlives the stream that produced it: ```cpp auto stream = xl::arrow_stream(workbook->sheet(0), 1, 10'000); @@ -330,7 +324,15 @@ std::array columns{ xl::i64_column("id", ids), xl::f64_column("value", values)}; -auto written = xl::write_columns("out.xlsx", XL_FORMAT_XLSX, columns); +auto written = xl::write_columns("out.xlsx", columns); +``` + +Both functions also write into memory: pass a `std::vector&` instead of a path and it +is replaced with the file's bytes. There is no extension to infer the format from, so name it: + +```cpp +std::vector bytes; +auto written = xl::write_sheet(bytes, rows, {.format = XL_FORMAT_XLSX}); ``` One constructor per column type: `i64_column`, `f64_column`, `bool_column`, `date_column`, @@ -343,24 +345,65 @@ is long enough for the row count before calling: the ABI takes it without a leng `(rows + 7) / 8` bytes on trust, so a short one would be a buffer overrun. On the struct side, declare the field `std::optional` and `write_sheet` builds the bitmap for you. -`xl::WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. -`XL_FORMAT_AUTO` is rejected — a file being created has no signature bytes to sniff — so -`xl::format_from_path` returning `XL_FORMAT_AUTO` for an unrecognized extension surfaces as a failed -write rather than a silently chosen format. +`xl::WriteOptions` sets the format, the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB +toggles. Its `format` defaults to `XL_FORMAT_AUTO`, which takes the format from the path's +extension (.xlsx, .xlsb, .xls or .csv, in any case); an unrecognized extension, or AUTO with a vector +destination, fails the write rather than silently choosing a format. `write_sheet` walks the range once and appends each field to its own column buffer, with the per-field dispatch resolved at compile time. That transpose is the only copy it makes, and it is what the ABI's columnar shape costs a row-shaped caller; `write_columns` pays nothing. +### Writing cell by cell + +`xl::WriterHandle` writes a workbook as calls arrive, sheet by sheet, without holding it in memory +first. Call `finish()` at the end: destruction finishes too, but cannot report a failure. Opened over +a `std::vector&`, the handle fills the vector when it finishes. + +```cpp +auto writer = xl::WriterHandle::open("out.xlsx").value(); +writer.start_sheet("Totals"); +writer.start_row(); +writer.write(std::string_view("widgets")); +writer.write(int64_t{7}); +writer.write(std::optional{}); // a blank cell +writer.end_row(); +writer.end_sheet(); +if (auto done = writer.finish(); !done) { /* done.error().message */ } +``` + +Every call returns `std::expected`; one out of order fails instead of corrupting the +file. + +## Aggregating a CSV in parallel + +`xl::aggregate_csv` folds a CSV file or buffer across worker threads. Each partition starts from +a copy of the seed, `accumulate` sees each record, and `combine` merges partitions: + +```cpp +struct RowCount +{ + int64_t rows = 0; + int32_t accumulate(xl::RowView) noexcept { ++rows; return XL_OK; } + int32_t combine(RowCount &next) noexcept { rows += next.rows; return XL_OK; } +}; + +auto counted = xl::aggregate_csv("big.csv", RowCount{}, {.csv_delimiter = ';'}, /*header_row*/ 1); +``` + +The dialect (`csv_delimiter`, `csv_quote`, `csv_detect_bom`) and `max_cell_bytes` come from +`xl::OpenOptions`; the header row and the degree of parallelism (0 = every processor) follow it. + ## Bounds and ABI `TableView::operator[]` is unchecked, like `std::vector`'s. Use `TableView::at(row)`, which returns `std::optional` and is `nullopt` outside `[0, size())`. -`xl::Workbook::open`/`open_memory` first check the loaded library's `xl_abi_version()` against the -`XL_ABI_VERSION` this header was compiled against, and fail with an explanatory `xl::Error` rather -than reading native memory through a layout that may have changed. `xl::abi_version()` exposes the -loaded revision directly. +Every entry point that starts native work (`xl::Workbook::open`, `write_columns`, `write_sheet`, +`WriterHandle::open`, `encrypt_package` and `aggregate_csv`) first checks the loaded library's +`xl_abi_version()` against the `XL_ABI_VERSION` this header was compiled against, and fails with an +explanatory `xl::Error` rather than reading native memory through a layout that may have changed. +`xl::abi_version()` exposes the loaded revision directly. ## Benchmarks @@ -370,12 +413,12 @@ on Windows 10 (22H2), 16 logical CPUs @ 3.39 GHz, MSVC 19.51 (Release), Google B | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| -| `open` (`open_memory`) | 24.0 µs | 1.17 ms | +| `open` (from memory) | 24.0 µs | 1.17 ms | | `parse_sheet` (4 or 6 bound columns) | 47.8 µs | 24.7 ms | | `typed_reader`, batches of 1,000 / 10,000 rows | - | 23.8 ms / 23.8 ms | | `infer_schema` (sample 100 / 1,000 rows) | 75.6 µs | 579.1 µs | -`open` here is `open_memory`, which copies the caller's buffer before returning (the caller may free +`open` here is the byte-span overload, which copies the caller's buffer before returning (the caller may free it afterwards), so it scales with file size: 1.17 ms is mostly copying the 3.7 MB fixture. Opening by path reads only the package directory and stays nearly flat — see the Rust suite below. `parse_sheet` scales linearly with rows × columns, at roughly 380 ns/row here. Draining the same diff --git a/cpp/benchmarks/benchmark_compare.cpp b/cpp/benchmarks/benchmark_compare.cpp index 39fdb096..dc183f7e 100644 --- a/cpp/benchmarks/benchmark_compare.cpp +++ b/cpp/benchmarks/benchmark_compare.cpp @@ -103,7 +103,7 @@ static void BM_ExcelReader_Xlsx_Full(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); @@ -135,7 +135,7 @@ static void BM_ExcelReader_Xlsx_ParseOnly(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); diff --git a/cpp/benchmarks/benchmark_csv_compare.cpp b/cpp/benchmarks/benchmark_csv_compare.cpp index 211faf13..239f3e07 100644 --- a/cpp/benchmarks/benchmark_csv_compare.cpp +++ b/cpp/benchmarks/benchmark_csv_compare.cpp @@ -202,7 +202,7 @@ static void BM_ExcelReader_Csv_Full(benchmark::State &state) const std::vector buffer = read_fixture(); for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer, XL_FORMAT_CSV); + auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); @@ -253,7 +253,7 @@ static void BM_ExcelReader_Csv_Cells(benchmark::State &state) const std::vector buffer = read_fixture(); for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer, XL_FORMAT_CSV); + auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message); diff --git a/cpp/benchmarks/benchmark_workbook.cpp b/cpp/benchmarks/benchmark_workbook.cpp index a80e4e5c..f971d13e 100644 --- a/cpp/benchmarks/benchmark_workbook.cpp +++ b/cpp/benchmarks/benchmark_workbook.cpp @@ -95,7 +95,7 @@ static void BM_Open(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); benchmark::DoNotOptimize(workbook); if (!workbook.has_value()) { @@ -117,7 +117,7 @@ static void BM_ParseSheet(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -147,7 +147,7 @@ static void BM_InferSchema(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -177,7 +177,7 @@ static void BM_Open_Large(benchmark::State &state) } for (auto _ : state) { - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); benchmark::DoNotOptimize(workbook); if (!workbook.has_value()) { @@ -199,7 +199,7 @@ static void BM_ParseSheet_Large(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -229,7 +229,7 @@ static void BM_TypedReaderBatched_Large(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); @@ -256,7 +256,7 @@ static void BM_TypedReaderBatched_Large(benchmark::State &state) benchmark::DoNotOptimize(rows); } } -BENCHMARK(BM_TypedReaderBatched_Large)->Arg(1000)->Arg(10000)->Arg(0); +BENCHMARK(BM_TypedReaderBatched_Large)->Arg(1000)->Arg(10000); static void BM_InferSchema_Large(benchmark::State &state) { @@ -269,7 +269,7 @@ static void BM_InferSchema_Large(benchmark::State &state) for (auto _ : state) { state.PauseTiming(); - auto workbook = xl::Workbook::open_memory(buffer_result.value(), XL_FORMAT_XLSB); + auto workbook = xl::Workbook::open(buffer_result.value(), {.format = XL_FORMAT_XLSB}); if (!workbook.has_value()) { state.SkipWithError(workbook.error().message.c_str()); diff --git a/cpp/benchmarks/benchmark_write.cpp b/cpp/benchmarks/benchmark_write.cpp index 40aea04c..206197dc 100644 --- a/cpp/benchmarks/benchmark_write.cpp +++ b/cpp/benchmarks/benchmark_write.cpp @@ -51,7 +51,7 @@ static const std::vector &fixture_rows() std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_LARGE_FIXTURE_PATH); std::abort(); } - return table->to_vector(); + return std::vector(table->begin(), table->end()); }(); return rows; } @@ -89,7 +89,7 @@ static void BM_WriteSheet(benchmark::State &state) const std::filesystem::path path = bench_path("sheet.xlsx"); for (auto _ : state) { - auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + auto result = xl::write_sheet(path.string(), rows); benchmark::DoNotOptimize(result); if (!result.has_value()) { @@ -158,7 +158,7 @@ static void BM_WriteColumns(benchmark::State &state) const std::filesystem::path path = bench_path("columns.xlsx"); for (auto _ : state) { - auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + auto result = xl::write_columns(path.string(), columns); benchmark::DoNotOptimize(result); if (!result.has_value()) { diff --git a/cpp/benchmarks/benchmark_write_compare.cpp b/cpp/benchmarks/benchmark_write_compare.cpp index 0e9eb191..4921631c 100644 --- a/cpp/benchmarks/benchmark_write_compare.cpp +++ b/cpp/benchmarks/benchmark_write_compare.cpp @@ -77,7 +77,7 @@ namespace { static const std::vector rows = [] { - auto workbook = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH, XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH, {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { std::fprintf(stderr, "missing or unreadable fixture %s\n", EXCELREADER_XLSX_FIXTURE_PATH); @@ -89,7 +89,7 @@ namespace std::fprintf(stderr, "fixture %s parsed to zero rows\n", EXCELREADER_XLSX_FIXTURE_PATH); std::abort(); } - return table->to_vector(); + return std::vector(table->begin(), table->end()); }(); return rows; } @@ -132,7 +132,7 @@ static void BM_ExcelReader_WriteSheet(benchmark::State &state) const std::filesystem::path path = bench_path("sheet.xlsx"); for (auto _ : state) { - auto result = xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows); + auto result = xl::write_sheet(path.string(), rows); if (!result.has_value()) { state.SkipWithError(result.error().message.c_str()); @@ -216,7 +216,7 @@ static void BM_ExcelReader_WriteColumns(benchmark::State &state) const std::filesystem::path path = bench_path("columns.xlsx"); for (auto _ : state) { - auto result = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + auto result = xl::write_columns(path.string(), columns); if (!result.has_value()) { state.SkipWithError(result.error().message.c_str()); diff --git a/cpp/tests/aggregate.cpp b/cpp/tests/aggregate.cpp index bc0841a7..b4c72147 100644 --- a/cpp/tests/aggregate.cpp +++ b/cpp/tests/aggregate.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -64,15 +65,19 @@ namespace static_assert(xl::CsvAccumulator, "RowCounter must satisfy xl::CsvAccumulator"); - template - concept SeedAccepted = requires(std::span data, Seed &seed) { - xl::aggregate_csv_memory(data, seed); + struct MoveOnly + { + std::unique_ptr state; + int32_t accumulate(xl::RowView) noexcept { return XL_OK; } + int32_t combine(MoveOnly &) noexcept { return XL_OK; } }; - static_assert(SeedAccepted, - "a const-callable seed must be accepted"); - static_assert(!SeedAccepted, - "a mutable seed must fail the constraint cleanly, not hard-error inside a shim"); + template + concept Aggregatable = requires(std::span data) { xl::aggregate_csv(data); }; + + static_assert(Aggregatable, "a copyable accumulator must be accepted"); + static_assert(!Aggregatable, + "every partition copies the seed, so a move-only accumulator must fail the constraint cleanly"); constexpr int64_t kLargeRowCount = 800'000; @@ -91,13 +96,9 @@ namespace static int test_aggregate_csv_memory() { - xl::CsvParallelOptions options{}; - options.header_row = 1; - options.degree_of_parallelism = 1; - std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data, &options); - CHECK(result.has_value(), "aggregate_csv_memory must succeed on a well-formed in-memory CSV"); + auto result = xl::aggregate_csv(data, RowCounter{}, {}, 1, 1); + CHECK(result.has_value(), "aggregate_csv must succeed on a well-formed in-memory CSV"); CHECK(result->rows == 3, "the fixture has 3 data rows after the header"); return 0; } @@ -111,14 +112,10 @@ static int test_aggregate_csv_file() out.write(kCsv.data(), static_cast(kCsv.size())); } - xl::CsvParallelOptions options{}; - options.header_row = 1; - options.degree_of_parallelism = 1; - - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 1, 1); std::filesystem::remove(path); - CHECK(result.has_value(), "aggregate_csv_file must succeed on a well-formed CSV file"); + CHECK(result.has_value(), "aggregate_csv must succeed on a well-formed CSV file"); CHECK(result->rows == 3, "the fixture has 3 data rows after the header"); return 0; } @@ -126,8 +123,8 @@ static int test_aggregate_csv_file() static int test_aggregate_default_options() { std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data); - CHECK(result.has_value(), "aggregate_csv_memory must succeed with default (null) options"); + auto result = xl::aggregate_csv(data); + CHECK(result.has_value(), "aggregate_csv must succeed with default options"); CHECK(result->rows == 4, "with no header configured, all 4 rows are data rows"); return 0; } @@ -138,13 +135,10 @@ static int test_aggregate_csv_file_partitioned() std::filesystem::temp_directory_path() / "excelreader-cpp-aggregate-large.csv"; write_repeated_rows(path, kLargeRowCount); - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 8; - - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 0, 8); std::filesystem::remove(path); - CHECK(result.has_value(), "aggregate_csv_file must succeed on a large multi-partition CSV file"); + CHECK(result.has_value(), "aggregate_csv must succeed on a large multi-partition CSV file"); CHECK(result->rows == kLargeRowCount, "every one of the generated rows must be counted exactly once"); CHECK(result->combines >= 1, @@ -173,9 +167,7 @@ static int test_aggregate_quoted_records_straddling_chunks() out.write(content.data(), static_cast(content.size())); } - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 8; - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 0, 8); std::filesystem::remove(path); CHECK(result.has_value(), "a file of quoted multi-line records must aggregate"); @@ -194,14 +186,14 @@ static int test_aggregate_reports_a_throwing_accumulate() }; std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data); + auto result = xl::aggregate_csv(data); CHECK(!result.has_value(), "a throwing accumulate must produce an error, not a value"); CHECK(std::string_view(result.error().message).find("threw") != std::string_view::npos, "the error must say a callback threw"); return 0; } -static int test_aggregate_with_a_seed_callable() +static int test_aggregate_with_a_seed() { struct Summer { @@ -226,14 +218,12 @@ static int test_aggregate_with_a_seed_callable() const int64_t step = 10; std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - xl::CsvParallelOptions options{}; - options.header_row = 1; - auto result = xl::aggregate_csv_memory(data, [step] { return Summer{0, step}; }, &options); - CHECK(result.has_value(), "aggregate_csv_memory with a seed callable must succeed"); + auto result = xl::aggregate_csv(data, Summer{0, step}, {}, 1); + CHECK(result.has_value(), "aggregate_csv with a seed must succeed"); CHECK(result->total == 30, "3 data rows at step 10 must total 30"); - auto no_options = xl::aggregate_csv_memory(data, [step] { return Summer{0, step}; }); - CHECK(no_options.has_value(), "a seed callable with default (null) options must succeed"); + auto no_options = xl::aggregate_csv(data, Summer{0, step}); + CHECK(no_options.has_value(), "a seed with default options must succeed"); CHECK(no_options->total == 40, "with no header configured, all 4 rows are data rows"); const std::filesystem::path path = @@ -242,9 +232,9 @@ static int test_aggregate_with_a_seed_callable() std::ofstream out(path, std::ios::binary); out.write(kCsv.data(), static_cast(kCsv.size())); } - auto from_file = xl::aggregate_csv_file(path.string(), [step] { return Summer{0, step}; }); + auto from_file = xl::aggregate_csv(path.string(), Summer{0, step}); std::filesystem::remove(path); - CHECK(from_file.has_value(), "aggregate_csv_file with a seed callable must succeed"); + CHECK(from_file.has_value(), "aggregate_csv of a file with a seed must succeed"); CHECK(from_file->total == 40, "with no header configured, all 4 rows are data rows"); return 0; } @@ -257,11 +247,8 @@ static int test_aggregate_reports_a_callback_status_of_int32_max() int32_t combine(ReturnsIntMax &) noexcept { return XL_OK; } }; - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 1; - std::span data(reinterpret_cast(kCsv.data()), kCsv.size()); - auto result = xl::aggregate_csv_memory(data, &options); + auto result = xl::aggregate_csv(data, {}, {}, 0, 1); CHECK(!result.has_value(), "a nonzero callback status must abort the run"); CHECK(result.error().code == (std::numeric_limits::max)(), "the caller's own status must come back verbatim"); @@ -283,10 +270,7 @@ static int test_aggregate_destroys_every_state() g_constructed.store(0); g_destructed.store(0); { - xl::CsvParallelOptions options{}; - options.degree_of_parallelism = 8; - - auto result = xl::aggregate_csv_file(path.string(), &options); + auto result = xl::aggregate_csv(path.string(), RowCounter{}, {}, 0, 8); CHECK(result.has_value(), "the run must succeed"); CHECK(result->combines >= 1, "the fixture must really be split, or free_state never runs and the balance is " @@ -300,6 +284,35 @@ static int test_aggregate_destroys_every_state() return 0; } +static int test_aggregate_takes_the_dialect_from_open_options() +{ + struct CellCounter + { + int64_t cells = 0; + + int32_t accumulate(xl::RowView row) noexcept + { + cells += static_cast(row.size()); + return XL_OK; + } + + int32_t combine(CellCounter &next) noexcept + { + cells += next.cells; + return XL_OK; + } + }; + + constexpr std::string_view semicolons = "a;b\n1;2\n3;4\n"; + std::span data(reinterpret_cast(semicolons.data()), semicolons.size()); + auto split = xl::aggregate_csv(data, CellCounter{}, {.csv_delimiter = ';'}, 1, 1); + CHECK(split.has_value() && split->cells == 4, "csv_delimiter must split each data row into two cells"); + + auto unsplit = xl::aggregate_csv(data, CellCounter{}, {}, 1, 1); + CHECK(unsplit.has_value() && unsplit->cells == 2, "the default delimiter leaves each row one cell"); + return 0; +} + int main() { if (int failed = test_aggregate_csv_memory()) @@ -330,7 +343,11 @@ int main() { return failed; } - if (int failed = test_aggregate_with_a_seed_callable()) + if (int failed = test_aggregate_with_a_seed()) + { + return failed; + } + if (int failed = test_aggregate_takes_the_dialect_from_open_options()) { return failed; } diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp index f42ca3c8..6c53074b 100644 --- a/cpp/tests/arrow.cpp +++ b/cpp/tests/arrow.cpp @@ -34,8 +34,8 @@ struct xl::ExcelMapper int main() { - xl::OpenOptions options{.prefetch_decompression = 1}; - auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + const xl::OpenOptions options{.format = XL_FORMAT_XLSB, .prefetch_decompression = XL_OPT_TRUE}; + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(workbook.has_value(), "xl::Workbook::open must succeed on the RealExcel.xlsb fixture"); auto table = xl::parse_arrow(workbook->sheet(0)); @@ -59,7 +59,7 @@ int main() CHECK(!failed.has_value(), "xl::parse_arrow must fail for an out-of-range header_row"); { - auto stream_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto stream_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(stream_workbook.has_value(), "opening a workbook for the Arrow stream must succeed"); auto stream = xl::arrow_stream(stream_workbook->sheet(0), 1, 8); @@ -68,6 +68,7 @@ int main() auto schema = stream->schema(); CHECK(schema.has_value(), "the stream must report a schema"); CHECK(schema->schema.release != nullptr, "the schema must be a live, owned struct"); + CHECK(schema->array.release == nullptr, "schema() must leave the table's array empty"); CHECK(std::strcmp(schema->schema.format, "+s") == 0, "the stream schema must be a struct array"); CHECK(schema->schema.n_children == 2, "the stream schema must have two child columns"); @@ -83,6 +84,8 @@ int main() } ++batches; CHECK((*batch)->array.length <= 8, "no batch may exceed the requested batch size"); + CHECK((*batch)->schema.release != nullptr && (*batch)->schema.n_children == 2, + "every batch must carry its own schema"); rows += (*batch)->array.length; } CHECK(rows == 100, "the stream must deliver all 100 data rows"); @@ -93,10 +96,10 @@ int main() } { - auto abandoned_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto abandoned_workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(abandoned_workbook.has_value(), "opening a workbook for the abandonment test must succeed"); - xl::ArrowArrayGuard kept; + xl::ArrowTable kept; { auto stream = xl::arrow_stream(abandoned_workbook->sheet(0), 1, 4); CHECK(stream.has_value(), "the abandoned stream must open"); @@ -107,39 +110,27 @@ int main() CHECK(kept.array.length == 4, "a batch must outlive the stream that produced it"); kept.release(); kept.release(); - CHECK(kept.array.release == nullptr, "releasing must null the release callback"); + CHECK(kept.array.release == nullptr && kept.schema.release == nullptr, "releasing must null both release callbacks"); } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); - CHECK(wb.has_value(), "opening a workbook for the batch_size=0 stream must succeed"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, 0); - CHECK(stream.has_value(), "xl::arrow_stream with batch_size=0 must succeed"); - auto batch = stream->next(); - CHECK(batch.has_value() && batch->has_value(), "batch_size=0 must yield one batch"); - CHECK((*batch)->array.length == 100, "the one batch must hold all 100 rows"); - auto end = stream->next(); - CHECK(end.has_value() && !end->has_value(), "the stream ends after the single batch"); - } - - { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); - CHECK(wb.has_value(), "opening a workbook for the negative-batch-size stream must succeed"); - auto stream = xl::arrow_stream(wb->sheet(0), 1, -1); - CHECK(!stream.has_value(), "a negative batch size must be rejected"); - if (!stream.has_value()) + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); + CHECK(wb.has_value(), "opening a workbook for the non-positive-batch-size stream must succeed"); + for (int64_t batch_size : {int64_t{0}, int64_t{-1}}) { + auto stream = xl::arrow_stream(wb->sheet(0), 1, batch_size); + CHECK(!stream.has_value(), "a batch size below 1 must be rejected"); CHECK(stream.error().code == XL_INVALID_ARGUMENT, "the rejection is XL_INVALID_ARGUMENT"); } } { - auto whole_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto whole_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(whole_wb.has_value(), "opening a workbook for the whole-sheet comparison must succeed"); auto whole = xl::parse_arrow(whole_wb->sheet(0)); CHECK(whole.has_value(), "the whole-sheet parse_arrow must succeed for comparison"); - auto stream_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto stream_wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(stream_wb.has_value(), "opening a workbook for the streamed comparison must succeed"); auto stream = xl::arrow_stream(stream_wb->sheet(0), 1, 4); CHECK(stream.has_value(), "xl::arrow_stream at batch_size=4 must succeed"); @@ -177,7 +168,7 @@ int main() } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(wb.has_value(), "opening a workbook for the foreign-read test must succeed"); auto stream = xl::arrow_stream(wb->sheet(0), 1, 4); CHECK(stream.has_value(), "the stream opens before the foreign read"); @@ -192,7 +183,7 @@ int main() } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(wb.has_value(), "opening a workbook for the reader-then-stream test must succeed"); auto reader = xl::typed_reader(wb->sheet(0), 1, 8); CHECK(reader.has_value(), "the typed reader opens first"); @@ -200,7 +191,7 @@ int main() CHECK(stream.has_value(), "an Arrow stream opens while a typed reader is live"); } { - auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(wb.has_value(), "opening a workbook for the stream-then-reader test must succeed"); auto stream = xl::arrow_stream(wb->sheet(0), 1, 8); CHECK(stream.has_value(), "the Arrow stream opens first"); diff --git a/cpp/tests/batch.cpp b/cpp/tests/batch.cpp index d8cc1b2c..a1fbcd0d 100644 --- a/cpp/tests/batch.cpp +++ b/cpp/tests/batch.cpp @@ -65,8 +65,7 @@ namespace void test_batches_equal_whole_sheet(const std::vector &expected) { const int64_t rows = static_cast(expected.size()); - for (int64_t batch_size : {int64_t{1}, int64_t{7}, int64_t{8}, int64_t{9}, int64_t{1000}, - int64_t{0}, rows + 10}) + for (int64_t batch_size : {int64_t{1}, int64_t{7}, int64_t{8}, int64_t{9}, int64_t{1000}, rows + 10}) { auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH); check(workbook.has_value(), "open the fixture for a batched read"); @@ -92,10 +91,7 @@ namespace break; } ++batches; - if (batch_size > 0) - { - check((*batch)->size() <= batch_size, "no batch exceeds the requested size"); - } + check((*batch)->size() <= batch_size, "no batch exceeds the requested size"); for (Row row : **batch) { actual.emplace_back(std::string(row.Coluna1), row.Coluna3); @@ -104,7 +100,7 @@ namespace check(actual == expected, "batched rows match the whole-sheet read"); - const int64_t want = batch_size == 0 ? 1 : (rows + batch_size - 1) / batch_size; + const int64_t want = (rows + batch_size - 1) / batch_size; check(batches == want, "batch count matches ceil(rows / batch_size)"); } } @@ -151,19 +147,22 @@ namespace check(second.has_value(), "a second reader on the same workbook opens"); } - void test_negative_batch_size_is_rejected() + void test_non_positive_batch_size_is_rejected() { auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH); - check(workbook.has_value(), "open the fixture for the negative-batch test"); + check(workbook.has_value(), "open the fixture for the non-positive-batch test"); if (!workbook.has_value()) { return; } - auto reader = xl::typed_reader(workbook->sheet(0), 1, -1); - check(!reader.has_value(), "a negative batch size is rejected"); - if (!reader.has_value()) + for (int64_t batch_size : {int64_t{0}, int64_t{-1}}) { - check(reader.error().code == XL_INVALID_ARGUMENT, "the rejection is XL_INVALID_ARGUMENT"); + auto reader = xl::typed_reader(workbook->sheet(0), 1, batch_size); + check(!reader.has_value(), "a batch size below 1 is rejected"); + if (!reader.has_value()) + { + check(reader.error().code == XL_INVALID_ARGUMENT, "the rejection is XL_INVALID_ARGUMENT"); + } } } @@ -199,7 +198,7 @@ int main() test_batches_equal_whole_sheet(expected); test_range_for_matches_next(expected); test_second_reader_is_allowed(); - test_negative_batch_size_is_rejected(); + test_non_positive_batch_size_is_rejected(); test_foreign_read_leaves_the_reader_alone(); if (failures != 0) diff --git a/cpp/tests/package/main.cpp b/cpp/tests/package/main.cpp index 2aaed760..3e6f1c7a 100644 --- a/cpp/tests/package/main.cpp +++ b/cpp/tests/package/main.cpp @@ -4,7 +4,7 @@ int main() { - auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSX, nullptr); + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, {.format = XL_FORMAT_XLSX}); if (!workbook.has_value()) { std::fprintf(stderr, "FAIL: the installed package could not open %s\n", EXCELREADER_FIXTURE_PATH); diff --git a/cpp/tests/rows.cpp b/cpp/tests/rows.cpp index 587cfe85..655c457b 100644 --- a/cpp/tests/rows.cpp +++ b/cpp/tests/rows.cpp @@ -86,7 +86,7 @@ int main() out << "a\n" << std::string(200000, 'x') << "\n"; } - auto workbook = xl::Workbook::open(path.string(), XL_FORMAT_CSV); + auto workbook = xl::Workbook::open(path.string(), {.format = XL_FORMAT_CSV}); check(workbook.has_value(), "open the oversized CSV"); if (workbook.has_value()) { @@ -107,17 +107,25 @@ int main() check((*big)[0].type == xl::CellType::String, "the oversized cell is a string"); } } + + auto limited = xl::Workbook::open(path.string(), {.format = XL_FORMAT_CSV, .max_cell_bytes = 1000}); + check(limited.has_value(), "open the oversized CSV under a cell limit"); + if (limited.has_value()) + { + auto all = limited->sheet(0).read_all(); + check(!all.has_value(), "max_cell_bytes bounds a CSV cell too"); + } std::error_code ignored; std::filesystem::remove(path, ignored); } { auto workbook = xl::Workbook::open(fixture()); - check(workbook.has_value(), "open for read_all_decoded"); + check(workbook.has_value(), "open for read_all"); if (workbook.has_value()) { - auto decoded = workbook->sheet(0).read_all_decoded(); - check(decoded.has_value(), "read_all_decoded succeeds"); + auto decoded = workbook->sheet(0).read_all(); + check(decoded.has_value(), "read_all succeeds"); if (decoded.has_value()) { check(decoded->size() == rows.size(), "same row count as the cursor"); @@ -148,7 +156,7 @@ int main() while (cursor.has_value() && cursor->next_row().has_value()) { } - auto decoded = workbook->sheet(0).read_all_decoded(); + auto decoded = workbook->sheet(0).read_all(); check(decoded.has_value(), "a read after another cursor drained is not an error"); check(decoded.has_value() && decoded->size() == rows.size(), "and it still reads every row, from the start"); } @@ -158,7 +166,7 @@ int main() auto workbook = xl::Workbook::open(fixture()); if (workbook.has_value()) { - auto decoded = workbook->sheet(0).read_all_decoded(); + auto decoded = workbook->sheet(0).read_all(); check(decoded.has_value(), "open a set to move"); if (decoded.has_value()) { diff --git a/cpp/tests/sheets.cpp b/cpp/tests/sheets.cpp index ea302817..9942daef 100644 --- a/cpp/tests/sheets.cpp +++ b/cpp/tests/sheets.cpp @@ -25,7 +25,8 @@ namespace std::vector build(int sheet_count, int row_count) { - auto writer = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); + std::vector bytes; + auto writer = xl::WriterHandle::open(bytes, {.format = XL_FORMAT_XLSX}); check(writer.has_value(), "open the in-memory writer"); if (!writer.has_value()) { @@ -44,9 +45,8 @@ namespace } check(writer->end_sheet().has_value(), "end a sheet"); } - auto bytes = writer->bytes(); - check(bytes.has_value(), "read the written bytes"); - return bytes.has_value() ? std::move(*bytes) : std::vector{}; + check(writer->finish().has_value(), "finish the in-memory writer"); + return bytes; } std::vector drain(xl::RowCursor &cursor) @@ -77,7 +77,7 @@ int main() const std::vector two = build(2, 3); { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open the two-sheet workbook"); if (workbook.has_value()) { @@ -109,22 +109,15 @@ int main() auto negative = workbook->sheet(-1).rows(); check(!negative.has_value() && negative.error().code == XL_INVALID_ARGUMENT, "a negative sheet index is an invalid argument"); - auto decoded = workbook->sheet(1).read_all_decoded(); - check(decoded.has_value() && decoded->size() == 3, "read_all_decoded reads the whole named sheet"); - auto partial = workbook->sheet(1).rows(); - check(partial.has_value() && partial->next_row().has_value(), "open a cursor and read one row"); - if (partial.has_value()) - { - auto rest = partial->read_all_decoded(); - check(rest.has_value() && rest->size() == 2, "the cursor's read_all_decoded returns the rows it has left"); - } + auto decoded = workbook->sheet(1).read_all(); + check(decoded.has_value() && decoded->size() == 3, "read_all reads the whole named sheet"); auto schema = workbook->sheet(1).infer_schema(0, 10); check(schema.has_value() && schema->size() == 1, "infer_schema samples the named sheet"); } } { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for two cursors"); if (workbook.has_value()) { @@ -142,7 +135,7 @@ int main() } { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for the moved cursor"); if (workbook.has_value()) { @@ -151,7 +144,7 @@ int main() if (cursor.has_value()) { xl::RowCursor moved = std::move(*cursor); - check(cursor->handle() == nullptr, "a moved-from cursor holds nothing"); + check(cursor->next_row().error().code == XL_INVALID_HANDLE, "a moved-from cursor holds nothing"); check(drain(moved).size() == 3, "the moved-to cursor reads every row"); } check(read_sheet(workbook->sheet(0)).size() == 3, "the workbook still reads after the cursors are gone"); @@ -159,7 +152,7 @@ int main() } { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for the move-assigned cursor"); if (workbook.has_value()) { @@ -170,7 +163,7 @@ int main() { check(a->next_row().has_value(), "the overwritten cursor advances"); *a = std::move(*b); - check(b->handle() == nullptr, "a move-assigned-from cursor holds nothing"); + check(b->next_row().error().code == XL_INVALID_HANDLE, "a move-assigned-from cursor holds nothing"); const auto values = drain(*a); check(values.size() == 3 && values.front() == "s1-r0", "the move-assigned cursor reads the other sheet from the top"); } @@ -179,11 +172,11 @@ int main() { const std::vector empty_book = build(1, 0); - auto workbook = xl::Workbook::open_memory(empty_book); + auto workbook = xl::Workbook::open(empty_book); check(workbook.has_value(), "open the empty workbook"); if (workbook.has_value()) { - auto decoded = workbook->sheet(0).read_all_decoded(); + auto decoded = workbook->sheet(0).read_all(); check(decoded.has_value() && decoded->empty(), "an empty sheet decodes to no rows"); } } @@ -192,7 +185,7 @@ int main() std::optional survivor; std::optional stale; { - auto workbook = xl::Workbook::open_memory(two); + auto workbook = xl::Workbook::open(two); check(workbook.has_value(), "open for the outliving cursor"); if (workbook.has_value()) { @@ -220,7 +213,7 @@ int main() constexpr int sheet_count = 6; constexpr int row_count = 500; const std::vector many = build(sheet_count, row_count); - auto workbook = xl::Workbook::open_memory(many); + auto workbook = xl::Workbook::open(many); check(workbook.has_value(), "open the many-sheet workbook"); if (workbook.has_value()) { diff --git a/cpp/tests/smoke.cpp b/cpp/tests/smoke.cpp index 4dd245ee..7a4a980e 100644 --- a/cpp/tests/smoke.cpp +++ b/cpp/tests/smoke.cpp @@ -75,17 +75,14 @@ static int test_parse(xl::Workbook &workbook) static int test_sheets(xl::Workbook &workbook) { - auto count = workbook.sheet_count(); - CHECK(count.has_value(), "sheet_count must succeed"); - CHECK(*count >= 1, "the fixture has at least one sheet"); + auto sheets = workbook.sheets(); + CHECK(sheets.has_value(), "sheets must succeed"); + CHECK(!sheets->empty(), "the fixture has at least one sheet"); - auto names = workbook.sheet_names(); - CHECK(names.has_value(), "sheet_names must succeed"); - CHECK(names->size() == static_cast(*count), "sheet_names must return one name per sheet"); - - auto first = workbook.sheet_name_at(0); - CHECK(first.has_value(), "sheet_name_at(0) must succeed"); - CHECK(*first == names->front(), "sheet_name_at(0) must be the first of sheet_names"); + auto first = sheets->front().name(); + CHECK(first.has_value(), "the first sheet's name must be readable"); + auto same = workbook.sheet(0).name(); + CHECK(same.has_value() && *same == *first, "sheet(0) must be the first of sheets()"); CHECK(workbook.is_date1904().has_value(), "is_date1904 must succeed"); return 0; @@ -119,19 +116,18 @@ static std::string encrypted_fixture_path(std::string_view filename) static int test_password_open() { xl::OpenOptions options{}; - options.password("hunter2"); - auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), XL_FORMAT_AUTO, &options); + options.password = "hunter2"; + auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), options); CHECK(workbook.has_value(), "opening an encrypted workbook with the correct password must succeed"); - auto count = workbook->sheet_count(); - CHECK(count.has_value(), "sheet_count must succeed on the decrypted workbook"); - CHECK(*count > 0, "the decrypted workbook must have at least one sheet"); + auto sheets = workbook->sheets(); + CHECK(sheets.has_value(), "sheets must succeed on the decrypted workbook"); + CHECK(!sheets->empty(), "the decrypted workbook must have at least one sheet"); return 0; } static int test_password_required() { - xl::OpenOptions options{}; - auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), XL_FORMAT_AUTO, &options); + auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx")); CHECK(!workbook.has_value(), "opening an encrypted workbook without a password must fail"); CHECK(workbook.error().code == XL_STATUS_PASSWORD_REQUIRED, "the failure must report XL_STATUS_PASSWORD_REQUIRED"); @@ -140,9 +136,7 @@ static int test_password_required() static int test_password_incorrect() { - xl::OpenOptions options{}; - options.password("wrong"); - auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), XL_FORMAT_AUTO, &options); + auto workbook = xl::Workbook::open(encrypted_fixture_path("agile-aes256-sha512.xlsx"), {.password = "wrong"}); CHECK(!workbook.has_value(), "opening an encrypted workbook with the wrong password must fail"); CHECK(workbook.error().code == XL_STATUS_PASSWORD_INCORRECT, "the failure must report XL_STATUS_PASSWORD_INCORRECT"); @@ -153,14 +147,21 @@ int main() { CHECK(xl::abi_version() == XL_ABI_VERSION, "the linked native library must speak this header's ABI revision"); - xl::OpenOptions options{.prefetch_decompression = 1}; - auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, XL_FORMAT_XLSB, &options); + xl::OpenOptions options{.format = XL_FORMAT_XLSB, .prefetch_decompression = XL_OPT_TRUE}; + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options); CHECK(workbook.has_value(), "xl::Workbook::open must succeed on the RealExcel.xlsb fixture"); auto missing = xl::Workbook::open("does-not-exist.xlsx"); CHECK(!missing.has_value(), "opening a missing file must fail"); CHECK(!missing.error().message.empty(), "a failure must carry the native error detail"); + auto wrong_format = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, {.format = XL_FORMAT_XLSX}); + CHECK(!wrong_format.has_value(), "OpenOptions::format must reach the open: an XLSB is not an XLSX"); + + auto bad_limit = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, {.max_cell_bytes = -1}); + CHECK(!bad_limit.has_value() && bad_limit.error().code == XL_INVALID_ARGUMENT, + "an out-of-range limit must fail the open with XL_INVALID_ARGUMENT"); + if (int failed = test_sheets(*workbook)) { return failed; diff --git a/cpp/tests/sources.cpp b/cpp/tests/sources.cpp index a84afe35..76c265d9 100644 --- a/cpp/tests/sources.cpp +++ b/cpp/tests/sources.cpp @@ -127,29 +127,27 @@ namespace std::vector build_sheets(int sheet_count, int row_count) { - auto writer = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); - std::vector bytes; - if (!writer.has_value()) + std::vector written; { - return bytes; - } - for (int s = 0; s < sheet_count; ++s) - { - (void)writer->start_sheet("sheet" + std::to_string(s)); - for (int r = 0; r < row_count; ++r) + auto writer = xl::WriterHandle::open(written, {.format = XL_FORMAT_XLSX}); + if (!writer.has_value()) { - (void)writer->start_row(); - (void)writer->write("s" + std::to_string(s) + "-r" + std::to_string(r)); - (void)writer->end_row(); + return {}; + } + for (int s = 0; s < sheet_count; ++s) + { + (void)writer->start_sheet("sheet" + std::to_string(s)); + for (int r = 0; r < row_count; ++r) + { + (void)writer->start_row(); + (void)writer->write("s" + std::to_string(s) + "-r" + std::to_string(r)); + (void)writer->end_row(); + } + (void)writer->end_sheet(); } - (void)writer->end_sheet(); - } - auto written = writer->bytes(); - if (written.has_value()) - { - bytes.resize(written->size()); - std::memcpy(bytes.data(), written->data(), written->size()); } + std::vector bytes(written.size()); + std::memcpy(bytes.data(), written.data(), written.size()); return bytes; } @@ -171,7 +169,7 @@ namespace std::size_t row_count(const xl::Workbook &workbook) { - auto decoded = workbook.sheet(0).read_all_decoded(); + auto decoded = workbook.sheet(0).read_all(); return decoded.has_value() ? decoded->size() : 0; } } @@ -186,9 +184,9 @@ int main() Counters counters; { - auto workbook = xl::Workbook::open_source(std::make_unique(bytes, counters)); - check(workbook.has_value(), "open_source opens the fixture"); - check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_source reads every row"); + auto workbook = xl::Workbook::open(std::make_unique(bytes, counters)); + check(workbook.has_value(), "a Source opens the fixture"); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "a Source reads every row"); } check(counters.destroyed.load() == 1, "the source is destroyed exactly once"); @@ -197,29 +195,28 @@ int main() Counters uncached; { - xl::OpenOptions options; - options.source_block_size = -1; - auto workbook = xl::Workbook::open_source(std::make_unique(bytes, uncached), XL_FORMAT_XLSX, &options); - check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_source without the cache reads every row"); + auto workbook = xl::Workbook::open(std::make_unique(bytes, uncached), + {.format = XL_FORMAT_XLSX, .source_block_size = -1}); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "a Source without the cache reads every row"); } check(uncached.reads.load() > blocks, "without the cache every read reaches read_at"); { Counters failing; - auto workbook = xl::Workbook::open_source(std::make_unique(failing), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(std::make_unique(failing), {.format = XL_FORMAT_XLSX}); check(!workbook.has_value() && workbook.error().message.find("disk on fire") != std::string::npos, "the source's error message reaches the caller"); check(failing.destroyed.load() == 1, "a source whose read failed is destroyed exactly once"); } { Counters throwing; - auto workbook = xl::Workbook::open_source(std::make_unique(throwing), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(std::make_unique(throwing), {.format = XL_FORMAT_XLSX}); check(!workbook.has_value() && workbook.error().message.find("boom") != std::string::npos, "an exception from the source becomes an error"); check(throwing.destroyed.load() == 1, "a source that threw is destroyed exactly once"); } { - auto workbook = xl::Workbook::open_stream(std::make_unique(), XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(std::make_unique(), {.format = XL_FORMAT_XLSX}); check(!workbook.has_value() && workbook.error().message.find("pipe broke") != std::string::npos, "the stream's error message reaches the caller"); } @@ -228,8 +225,8 @@ int main() const std::vector many = build_sheets(sheet_count, 300); check(!many.empty(), "build the multi-sheet workbook"); Counters parallel_counters; - auto reference_many = xl::Workbook::open_memory(std::span(reinterpret_cast(many.data()), many.size())); - auto workbook = xl::Workbook::open_source(std::make_unique(many, parallel_counters), XL_FORMAT_XLSX); + auto reference_many = xl::Workbook::open(std::span(reinterpret_cast(many.data()), many.size())); + auto workbook = xl::Workbook::open(std::make_unique(many, parallel_counters), {.format = XL_FORMAT_XLSX}); check(reference_many.has_value() && workbook.has_value(), "open the multi-sheet workbook both ways"); if (reference_many.has_value() && workbook.has_value()) { @@ -259,12 +256,12 @@ int main() const std::string csv = "name,qty\nwidget,7\n"; std::vector csv_bytes(csv.size()); std::memcpy(csv_bytes.data(), csv.data(), csv.size()); - auto workbook = xl::Workbook::open_stream(std::make_unique(csv_bytes), XL_FORMAT_CSV); - check(workbook.has_value() && row_count(*workbook) == 2, "open_stream reads a CSV"); + auto workbook = xl::Workbook::open(std::make_unique(csv_bytes), {.format = XL_FORMAT_CSV}); + check(workbook.has_value() && row_count(*workbook) == 2, "an InputStream reads a CSV"); } { - auto workbook = xl::Workbook::open_stream(std::make_unique(bytes)); - check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_stream reads an XLSX"); + auto workbook = xl::Workbook::open(std::make_unique(bytes)); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "an InputStream reads an XLSX"); } if (failures != 0) diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index b4809485..8fc71950 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #define CHECK(cond, msg) \ do \ @@ -69,7 +70,7 @@ static int test_write_columns_round_trip() xl::timestamp_column("instante", instantes)}; const std::filesystem::path path = temp_path("columns.xlsx"); - auto written = xl::write_columns(path.string(), XL_FORMAT_XLSX, columns); + auto written = xl::write_columns(path.string(), columns); CHECK(written.has_value(), "write_columns must succeed"); { auto workbook = xl::Workbook::open(path.string()); @@ -114,16 +115,14 @@ static int test_encrypt_package_round_trip() const std::filesystem::path plain_path = temp_path("encrypt-plain.xlsx"); const std::filesystem::path encrypted_path = temp_path("encrypt-cipher.xlsx"); - auto written = xl::write_columns(plain_path.string(), XL_FORMAT_XLSX, columns); + auto written = xl::write_columns(plain_path.string(), columns); CHECK(written.has_value(), "write_columns must succeed"); auto encrypted = xl::encrypt_package(plain_path.string(), encrypted_path.string(), "hunter2"); CHECK(encrypted.has_value(), "encrypt_package must succeed"); { - xl::OpenOptions options; - options.password("hunter2"); - auto workbook = xl::Workbook::open(encrypted_path.string(), XL_FORMAT_AUTO, &options); + auto workbook = xl::Workbook::open(encrypted_path.string(), {.password = "hunter2"}); CHECK(workbook.has_value(), "the encrypted file must open with the right password"); auto table = xl::parse_sheet(workbook->sheet(0)); CHECK(table.has_value(), "the decrypted file must parse back"); @@ -147,7 +146,7 @@ static int test_encrypt_package_rejects_an_empty_password() { const std::filesystem::path plain_path = temp_path("encrypt-empty-pw.xlsx"); const std::array columns{xl::i64_column("inteiro", std::vector{1})}; - auto written = xl::write_columns(plain_path.string(), XL_FORMAT_XLSX, columns); + auto written = xl::write_columns(plain_path.string(), columns); CHECK(written.has_value(), "write_columns must succeed"); auto encrypted = xl::encrypt_package(plain_path.string(), temp_path("encrypt-empty-pw-out.xlsx").string(), ""); @@ -175,11 +174,12 @@ static int test_write_columns_to_memory_round_trip() xl::time_column("hora", horas), xl::timestamp_column("instante", instantes)}; - auto bytes = xl::write_columns_to_memory(XL_FORMAT_XLSX, columns); - CHECK(bytes.has_value(), "write_columns_to_memory must succeed"); - CHECK(!bytes->empty(), "write_columns_to_memory must return non-empty bytes"); + std::vector bytes; + CHECK(xl::write_columns(bytes, columns, {.format = XL_FORMAT_XLSX}).has_value(), + "write_columns into a vector must succeed"); + CHECK(!bytes.empty(), "write_columns must fill the vector"); - auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(bytes, {.format = XL_FORMAT_XLSX}); CHECK(workbook.has_value(), "the returned bytes must open"); auto table = xl::parse_sheet(workbook->sheet(0)); CHECK(table.has_value(), "the returned bytes must parse back"); @@ -199,25 +199,29 @@ static int test_write_columns_rejects_bad_input() const std::filesystem::path path = temp_path("rejected.xlsx"); std::array mismatched{xl::i64_column("a", two), xl::i64_column("b", three)}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, mismatched).has_value(), + CHECK(!xl::write_columns(path.string(), mismatched).has_value(), "columns of different lengths must be rejected"); std::array partial_header{xl::i64_column("a", two), xl::i64_column("", two)}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, partial_header).has_value(), + CHECK(!xl::write_columns(path.string(), partial_header).has_value(), "a partial header row must be rejected"); std::array short_bitmap{xl::i64_column("a", two)}; short_bitmap[0].validity = reinterpret_cast(two.data()); short_bitmap[0].validity_len = 0; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, short_bitmap).has_value(), + CHECK(!xl::write_columns(path.string(), short_bitmap).has_value(), "a validity bitmap shorter than the row count must be rejected"); std::array fine{xl::i64_column("a", two)}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_AUTO, fine).has_value(), - "XL_FORMAT_AUTO must be rejected: a new file has no signature bytes to sniff"); + CHECK(!xl::write_columns(temp_path("rejected.txt").string(), fine).has_value(), + "XL_FORMAT_AUTO with an unknown extension must be rejected"); + std::vector untouched{42}; + CHECK(!xl::write_columns(untouched, fine).has_value(), + "XL_FORMAT_AUTO must be rejected for memory: there is no extension to infer it from"); + CHECK(untouched.size() == 1 && untouched[0] == 42, "a failed write leaves the vector untouched"); std::span none{}; - CHECK(!xl::write_columns(path.string(), XL_FORMAT_XLSX, none).has_value(), + CHECK(!xl::write_columns(path.string(), none).has_value(), "an empty column set must be rejected"); std::filesystem::remove(path); @@ -226,35 +230,39 @@ static int test_write_columns_rejects_bad_input() static int test_write_options() { - xl::WriteOptions defaults{}; - xl_write_options raw = defaults.to_c(); - CHECK(raw.struct_size == static_cast(sizeof(xl_write_options)), "to_c must fill struct_size"); - CHECK(raw.sheet_name == nullptr, "an empty sheet_name must lower to NULL, meaning Sheet1"); - CHECK(raw.sheet_name_len == 0, "an empty sheet_name must lower to length 0"); - CHECK(raw.csv_delimiter == 0, "an unset csv_delimiter must lower to 0"); - CHECK(raw.date1904 == XL_OPT_DEFAULT, "an unset tri-state must lower to XL_OPT_DEFAULT"); - - xl::WriteOptions configured{}; - configured.sheet_name = "Dados"; - configured.csv_delimiter = ';'; - configured.use_shared_strings = XL_OPT_TRUE; - xl_write_options set = configured.to_c(); - CHECK(set.sheet_name_len == 5, "sheet_name_len must be the byte length"); - CHECK(set.sheet_name != nullptr, "a non-empty sheet_name must lower to a pointer"); - CHECK(set.csv_delimiter == ';', "csv_delimiter must pass through unchanged"); - CHECK(set.use_shared_strings == XL_OPT_TRUE, "use_shared_strings must pass through unchanged"); + const std::vector values{1, 2}; + const std::array columns{xl::i64_column("a", values), xl::i64_column("b", values)}; + + std::vector bytes; + CHECK(xl::write_columns(bytes, columns, {.format = XL_FORMAT_CSV, .csv_delimiter = ';'}).has_value(), + "writing a CSV into a vector must succeed"); + const std::string_view text(reinterpret_cast(bytes.data()), bytes.size()); + CHECK(text.find("a;b") != std::string_view::npos, "WriteOptions::csv_delimiter must reach the output"); return 0; } -static int test_format_from_path() +static int test_format_from_extension() { - CHECK(xl::format_from_path("out.xlsx") == XL_FORMAT_XLSX, ".xlsx must resolve to XL_FORMAT_XLSX"); - CHECK(xl::format_from_path("out.XLSB") == XL_FORMAT_XLSB, "the extension match must be case-insensitive"); - CHECK(xl::format_from_path("out.xls") == XL_FORMAT_XLS, ".xls must resolve to XL_FORMAT_XLS"); - CHECK(xl::format_from_path("out.csv") == XL_FORMAT_CSV, ".csv must resolve to XL_FORMAT_CSV"); - CHECK(xl::format_from_path("out.txt") == XL_FORMAT_AUTO, "an unknown extension must resolve to AUTO"); - CHECK(xl::format_from_path("out") == XL_FORMAT_AUTO, "no extension at all must resolve to AUTO"); - CHECK(xl::format_from_path("v1.2/report") == XL_FORMAT_AUTO, "a dot before the last separator is not an extension"); + const std::vector values{1, 2}; + const std::array columns{xl::i64_column("a", values)}; + const std::array, 4> cases{{{"ext.xlsx", XL_FORMAT_XLSX}, + {"ext.XLSB", XL_FORMAT_XLSB}, + {"ext.xls", XL_FORMAT_XLS}, + {"ext.csv", XL_FORMAT_CSV}}}; + for (const auto &[name, format] : cases) + { + const std::filesystem::path path = temp_path(name); + CHECK(xl::write_columns(path.string(), columns).has_value(), "a known extension must pick the format"); + { + auto workbook = xl::Workbook::open(path.string(), {.format = format}); + CHECK(workbook.has_value(), "the file must open as the format its extension names"); + auto rows = workbook->sheet(0).read_all(); + CHECK(rows.has_value() && rows->size() == 3, "the header and both rows must read back"); + } + std::filesystem::remove(path); + } + CHECK(!xl::write_columns(temp_path("no-extension").string(), columns).has_value(), + "a path without an extension must be rejected"); return 0; } @@ -278,7 +286,7 @@ static int test_bool_round_trip() const std::array columns{xl::bool_column("ativo", flags)}; const std::filesystem::path path = temp_path("bools.xlsx"); - CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + CHECK(xl::write_columns(path.string(), columns).has_value(), "writing a bool column must succeed"); { auto workbook = xl::Workbook::open(path.string()); @@ -316,7 +324,7 @@ static int test_optional_round_trip() const std::array columns{xl::i64_column("quantidade", valores, validity)}; const std::filesystem::path path = temp_path("nullable.xlsx"); - CHECK(xl::write_columns(path.string(), XL_FORMAT_XLSX, columns).has_value(), + CHECK(xl::write_columns(path.string(), columns).has_value(), "writing a nullable column must succeed"); { auto workbook = xl::Workbook::open(path.string()); @@ -420,10 +428,11 @@ static int test_write_sheet_to_memory_round_trip() std::chrono::system_clock::time_point{std::chrono::microseconds{1767312000000000}}, std::nullopt}}; - auto bytes = xl::write_sheet_to_memory(XL_FORMAT_XLSX, rows); - CHECK(bytes.has_value(), "write_sheet_to_memory must succeed"); + std::vector bytes; + CHECK(xl::write_sheet(bytes, rows, {.format = XL_FORMAT_XLSX}).has_value(), + "write_sheet into a vector must succeed"); - auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); + auto workbook = xl::Workbook::open(bytes); CHECK(workbook.has_value(), "the returned bytes must open"); auto table = xl::parse_sheet(workbook->sheet(0)); CHECK(table.has_value(), "the returned bytes must parse back"); @@ -444,14 +453,12 @@ static int test_write_sheet_options_and_csv() const std::vector rows{}; const std::filesystem::path path = temp_path("named.xlsx"); - xl::WriteOptions options{}; - options.sheet_name = "Dados"; - CHECK(xl::write_sheet(path.string(), XL_FORMAT_XLSX, rows, &options).has_value(), + CHECK(xl::write_sheet(path.string(), rows, {.format = XL_FORMAT_XLSX, .sheet_name = "Dados"}).has_value(), "writing an empty sheet with a custom name must succeed"); { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the written file must open"); - auto name = workbook->sheet_name_at(0); + auto name = workbook->sheet(0).name(); CHECK(name.has_value(), "the sheet name must be readable"); CHECK(*name == "Dados", "WriteOptions::sheet_name must reach the file"); } @@ -509,7 +516,7 @@ static int test_writer_handle() { auto workbook = xl::Workbook::open(path.string()); CHECK(workbook.has_value(), "the file xl_close_write_handle produced must open"); - auto name = workbook->sheet_name_at(0); + auto name = workbook->sheet(0).name(); CHECK(name.has_value() && *name == "Planilha1", "xl_start_sheet's name must reach the file"); auto table = xl::parse_sheet(workbook->sheet(0)); CHECK(table.has_value(), "the written file must parse back"); @@ -567,7 +574,7 @@ static int test_writer_handle_class_round_trip() { const std::filesystem::path path = temp_path("writer_handle_class.xlsx"); { - auto handle = xl::WriterHandle::open(path.string(), XL_FORMAT_XLSX); + auto handle = xl::WriterHandle::open(path.string()); CHECK(handle.has_value(), "WriterHandle::open must succeed"); CHECK(handle->start_sheet("Planilha1").has_value(), "start_sheet must succeed"); @@ -605,6 +612,8 @@ static int test_writer_handle_class_round_trip() CHECK(handle->end_row().has_value(), "end_row must succeed for the null-cell row"); CHECK(handle->end_sheet().has_value(), "end_sheet must succeed"); + CHECK(handle->finish().has_value(), "finish must succeed"); + CHECK(!handle->start_row().has_value(), "a finished handle accepts no more writes"); } { auto workbook = xl::Workbook::open(path.string()); @@ -632,44 +641,34 @@ static int test_writer_handle_class_round_trip() static int test_writer_handle_class_to_memory_round_trip() { - auto handle = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); - CHECK(handle.has_value(), "WriterHandle::open_memory must succeed"); + std::vector bytes; + auto handle = xl::WriterHandle::open(bytes, {.format = XL_FORMAT_XLSX}); + CHECK(handle.has_value(), "WriterHandle::open over a vector must succeed"); CHECK(handle->start_sheet("Dados").has_value(), "start_sheet must succeed"); CHECK(handle->start_row().has_value(), "start_row must succeed"); CHECK(handle->write(std::string_view("uma")).has_value(), "write(string_view) must succeed"); CHECK(handle->write(int64_t{3}).has_value(), "write(int64_t) must succeed"); CHECK(handle->end_row().has_value(), "end_row must succeed"); + CHECK(handle->end_sheet().has_value(), "end_sheet must succeed"); + CHECK(bytes.empty(), "the vector is filled only when the handle finishes"); - auto bytes = handle->bytes(); - CHECK(bytes.has_value(), "bytes() must succeed"); - CHECK(!bytes->empty(), "bytes() must return non-empty bytes"); + CHECK(handle->finish().has_value(), "finish must succeed"); + CHECK(!bytes.empty(), "finish must fill the vector"); + CHECK(handle->finish().has_value(), "a second finish does nothing"); - auto workbook = xl::Workbook::open_memory(*bytes, XL_FORMAT_XLSX); - CHECK(workbook.has_value(), "the bytes bytes() returned must open"); - auto sheet_name = workbook->sheet_name_at(0); + auto workbook = xl::Workbook::open(bytes, {.format = XL_FORMAT_XLSX}); + CHECK(workbook.has_value(), "the bytes finish wrote must open"); + auto sheet_name = workbook->sheet(0).name(); CHECK(sheet_name.has_value() && *sheet_name == "Dados", "start_sheet's name must reach the bytes"); - - auto bytes_again = handle->bytes(); - CHECK(bytes_again.has_value(), "a second bytes() call must still succeed"); - CHECK(*bytes_again == *bytes, "a second bytes() call must return the same content"); return 0; } -static int test_writer_handle_class_bytes_rejects_a_file_backed_handle() +static int test_writer_handle_class_rejects_auto_for_memory() { - const std::filesystem::path path = temp_path("writer_handle_class_file.xlsx"); - { - auto handle = xl::WriterHandle::open(path.string(), XL_FORMAT_XLSX); - CHECK(handle.has_value(), "WriterHandle::open must succeed"); - CHECK(handle->start_sheet("S").has_value(), "start_sheet must succeed"); - - auto bytes = handle->bytes(); - CHECK(!bytes.has_value(), "bytes() on a file-backed handle must fail"); - CHECK(bytes.error().code == XL_INVALID_ARGUMENT, - "bytes() on a file-backed handle must be XL_INVALID_ARGUMENT"); - } - std::filesystem::remove(path); + std::vector bytes; + auto handle = xl::WriterHandle::open(bytes); + CHECK(!handle.has_value(), "WriterHandle::open over a vector must reject XL_FORMAT_AUTO"); return 0; } @@ -679,7 +678,7 @@ int main() { return 1; } - if (test_format_from_path() != 0) + if (test_format_from_extension() != 0) { return 1; } @@ -743,7 +742,7 @@ int main() { return 1; } - if (test_writer_handle_class_bytes_rejects_a_file_backed_handle() != 0) + if (test_writer_handle_class_rejects_auto_for_memory() != 0) { return 1; } diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 67fb8abe..6194f1c0 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -71,7 +71,9 @@ namespace xl return buffer; } - } + struct Access; + + } /// @brief ABI revision of the loaded native library. @@ -136,7 +138,7 @@ namespace xl }; - /// @brief Reader limits and CSV dialect for the Workbook::open family. + /// @brief Format, reader limits and CSV dialect for Workbook::open and aggregate_csv. /// /// A field left at 0 or `XL_OPT_DEFAULT` keeps the library default; on/off fields take /// `XL_OPT_TRUE` or `XL_OPT_FALSE`. The `max_*` fields bound what a malformed or hostile file can @@ -144,12 +146,12 @@ namespace xl /// with `XL_INVALID_ARGUMENT`. struct OpenOptions { + int32_t format = XL_FORMAT_AUTO; ///< An `XL_FORMAT_*` constant. `XL_FORMAT_AUTO` detects XLSX, XLSB and XLS from the signature; CSV has none and needs `XL_FORMAT_CSV`, except through an InputStream, where AUTO reads anything else as CSV. + int32_t csv_sniff_dialect = XL_OPT_DEFAULT; ///< Infer delimiter and quote from the start of a CSV. Off by default. int32_t csv_delimiter = 0; ///< CSV field separator byte. Default `,`. int32_t csv_quote = 0; ///< CSV quote byte. Default `"`. int32_t csv_detect_bom = XL_OPT_DEFAULT; ///< Skip a leading UTF-8 byte-order mark. On by default. - int32_t csv_max_cell_bytes = 0; ///< For CSV, replaces `max_cell_bytes`. - int32_t csv_intern_strings = XL_OPT_DEFAULT; ///< For CSV, replaces `intern_strings`. int64_t max_total_decompressed_bytes = 0; ///< Bytes an XLSX/XLSB may inflate to, or an XLS workbook stream may declare. Default 512 MiB. int32_t max_cell_bytes = 0; ///< Largest cell value or buffered CSV record. Default 32 MiB. @@ -157,59 +159,26 @@ namespace xl int32_t max_zip_entries = 0; ///< Most entries an XLSX/XLSB archive may hold. Default 65,536. int32_t prefetch_decompression = XL_OPT_DEFAULT; ///< Inflate XLSX/XLSB sheet data on a background thread ahead of parsing. Off by default. int32_t intern_strings = XL_OPT_DEFAULT; ///< Deduplicate repeated text outside the shared-string table. Off by default; pays off only for low-cardinality text. - int64_t source_cache_bytes = 0; ///< Block cache of Workbook::open_source. Default 64 MiB. - int64_t max_buffered_bytes = 0; ///< Most bytes Workbook::open_stream reads before an XLSX, XLSB or XLS opens. - int32_t source_block_size = 0; ///< Block size of Workbook::open_source's cache. Default 4 MiB; negative turns the cache off. + int64_t source_cache_bytes = 0; ///< Block cache of a Source. Default 64 MiB. + int64_t max_buffered_bytes = 0; ///< Most bytes an InputStream may supply before an XLSX, XLSB or XLS opens. + int32_t source_block_size = 0; ///< Block size of a Source's cache. Default 4 MiB; negative turns the cache off. int32_t verify_zip_checksums = XL_OPT_DEFAULT; ///< Check each XLSX/XLSB ZIP entry against its CRC-32. On by default; a mismatch fails the read. - std::string password_{}; ///< Set through password(). - - /// @brief Sets the password that unlocks an encrypted OOXML workbook (.xlsx/.xlsb/.xlsm). + /// @brief UTF-8 password that unlocks an encrypted OOXML workbook (.xlsx/.xlsb/.xlsm); empty for none. /// /// Open the workbook with `XL_FORMAT_AUTO`: `XL_FORMAT_XLSX` reads the encrypted container as a /// plain ZIP and fails before the password is checked. - /// @param value The password, as UTF-8. - /// @return `*this`, for chaining. - OpenOptions &password(std::string_view value) - { - password_ = std::string(value); - return *this; - } - - /// @brief These options as the C ABI's `xl_open_options`. - /// @return A struct pointing into this object, valid while it lives unchanged. - xl_open_options to_c() const noexcept - { - xl_open_options opts{}; - opts.struct_size = sizeof(xl_open_options); - opts.csv_sniff_dialect = csv_sniff_dialect; - opts.csv_delimiter = csv_delimiter; - opts.csv_quote = csv_quote; - opts.csv_detect_bom = csv_detect_bom; - opts.csv_max_cell_bytes = csv_max_cell_bytes; - opts.csv_intern_strings = csv_intern_strings; - opts.max_total_decompressed_bytes = max_total_decompressed_bytes; - opts.max_cell_bytes = max_cell_bytes; - opts.max_shared_string_bytes = max_shared_string_bytes; - opts.max_zip_entries = max_zip_entries; - opts.prefetch_decompression = prefetch_decompression; - opts.intern_strings = intern_strings; - opts.source_cache_bytes = source_cache_bytes; - opts.max_buffered_bytes = max_buffered_bytes; - opts.source_block_size = source_block_size; - opts.verify_zip_checksums = verify_zip_checksums; - opts.password = password_.empty() ? nullptr : reinterpret_cast(password_.data()); - opts.password_len = static_cast(password_.size()); - return opts; - } + std::string password{}; }; - /// @brief Settings for every writer. + /// @brief Format and settings for every writer. /// /// A field left at 0 or `XL_OPT_DEFAULT` keeps the library default. struct WriteOptions { + int32_t format = XL_FORMAT_AUTO; ///< An `XL_FORMAT_*` constant. `XL_FORMAT_AUTO` takes it from a path's extension (.xlsx, .xlsb, .xls or .csv, in any case) and fails for any other extension or when writing to memory. + std::string_view sheet_name{}; ///< Default `Sheet1`; ignored for CSV. Borrowed: must outlive the write. int32_t csv_delimiter = 0; ///< CSV field separator byte. Default `,`. @@ -217,65 +186,64 @@ namespace xl int32_t date1904 = XL_OPT_DEFAULT; ///< XLS/XLSB only: use the 1904 date system. Off by default. int32_t use_shared_strings = XL_OPT_DEFAULT; ///< XLSX/XLSB only: store repeated text once in a shared-string table. Off by default. + }; + + namespace detail + { + + inline xl_open_options to_c(const OpenOptions &options) noexcept + { + xl_open_options opts{}; + opts.struct_size = sizeof(xl_open_options); + opts.csv_sniff_dialect = options.csv_sniff_dialect; + opts.csv_delimiter = options.csv_delimiter; + opts.csv_quote = options.csv_quote; + opts.csv_detect_bom = options.csv_detect_bom; + opts.csv_max_cell_bytes = options.max_cell_bytes; + opts.csv_intern_strings = options.intern_strings; + opts.max_total_decompressed_bytes = options.max_total_decompressed_bytes; + opts.max_cell_bytes = options.max_cell_bytes; + opts.max_shared_string_bytes = options.max_shared_string_bytes; + opts.max_zip_entries = options.max_zip_entries; + opts.prefetch_decompression = options.prefetch_decompression; + opts.intern_strings = options.intern_strings; + opts.source_cache_bytes = options.source_cache_bytes; + opts.max_buffered_bytes = options.max_buffered_bytes; + opts.source_block_size = options.source_block_size; + opts.verify_zip_checksums = options.verify_zip_checksums; + opts.password = options.password.empty() ? nullptr : reinterpret_cast(options.password.data()); + opts.password_len = static_cast(options.password.size()); + return opts; + } - /// @brief These options as the C ABI's `xl_write_options`. - /// @return A struct pointing into this object, valid while it lives unchanged. - xl_write_options to_c() const noexcept + inline xl_write_options to_c(const WriteOptions &options) noexcept { xl_write_options opts{}; opts.struct_size = sizeof(xl_write_options); - opts.sheet_name_len = static_cast(sheet_name.size()); - opts.sheet_name = sheet_name.empty() + opts.sheet_name_len = static_cast(options.sheet_name.size()); + opts.sheet_name = options.sheet_name.empty() ? nullptr - : reinterpret_cast(sheet_name.data()); - opts.csv_delimiter = csv_delimiter; - opts.csv_quote = csv_quote; - opts.date1904 = date1904; - opts.use_shared_strings = use_shared_strings; + : reinterpret_cast(options.sheet_name.data()); + opts.csv_delimiter = options.csv_delimiter; + opts.csv_quote = options.csv_quote; + opts.date1904 = options.date1904; + opts.use_shared_strings = options.use_shared_strings; return opts; } - }; - - /// @brief Settings for aggregate_csv_file and aggregate_csv_memory. - /// - /// A zero field keeps the library default. - struct CsvParallelOptions - { - int32_t degree_of_parallelism = 0; ///< Worker threads; 0 uses every processor, 1 runs sequentially. - int32_t header_row = 0; ///< 1-based header row, not aggregated; 0 means none. - int32_t delimiter = 0; ///< Field separator byte. Default `,`. - int32_t quote = 0; ///< Quote byte. Default `"`. - int32_t detect_bom = XL_OPT_DEFAULT; ///< Skip a leading UTF-8 byte-order mark. On by default. - int32_t max_cell_bytes = 0; ///< Largest field or buffered record. Default 32 MiB. - /// @brief These options as the C ABI's `xl_csv_parallel_options`. - xl_csv_parallel_options to_c() const noexcept + inline xl_csv_parallel_options to_c(const OpenOptions &options, int32_t header_row, + int32_t degree_of_parallelism) noexcept { xl_csv_parallel_options opts{}; opts.struct_size = sizeof(xl_csv_parallel_options); opts.degree_of_parallelism = degree_of_parallelism; opts.header_row = header_row; - opts.delimiter = delimiter; - opts.quote = quote; - opts.detect_bom = detect_bom; - opts.max_cell_bytes = max_cell_bytes; + opts.delimiter = options.csv_delimiter; + opts.quote = options.csv_quote; + opts.detect_bom = options.csv_detect_bom; + opts.max_cell_bytes = options.max_cell_bytes; return opts; } - }; - - namespace detail - { - - template - inline const Raw *lower_options(const Opts *options, Raw &storage) noexcept - { - if (options == nullptr) - { - return nullptr; - } - storage = options->to_c(); - return &storage; - } constexpr bool ends_with_ci(std::string_view text, std::string_view suffix) noexcept { @@ -296,35 +264,31 @@ namespace xl return true; } - } - - /// @brief The format a file name's extension names. - /// @param path File name or path. Only its extension is read: .xlsx, .xlsb, .xls or .csv, in any case. - /// @return The matching `XL_FORMAT_*` constant, or `XL_FORMAT_AUTO` when none matches, which every - /// writer rejects. - constexpr int32_t format_from_path(std::string_view path) noexcept - { - const size_t separator = path.find_last_of("/\\"); - const std::string_view name = (separator == std::string_view::npos) - ? path - : path.substr(separator + 1); - if (detail::ends_with_ci(name, ".xlsx")) + constexpr int32_t format_from_path(std::string_view path) noexcept { - return XL_FORMAT_XLSX; - } - if (detail::ends_with_ci(name, ".xlsb")) - { - return XL_FORMAT_XLSB; - } - if (detail::ends_with_ci(name, ".xls")) - { - return XL_FORMAT_XLS; - } - if (detail::ends_with_ci(name, ".csv")) - { - return XL_FORMAT_CSV; + const size_t separator = path.find_last_of("/\\"); + const std::string_view name = (separator == std::string_view::npos) + ? path + : path.substr(separator + 1); + if (ends_with_ci(name, ".xlsx")) + { + return XL_FORMAT_XLSX; + } + if (ends_with_ci(name, ".xlsb")) + { + return XL_FORMAT_XLSB; + } + if (ends_with_ci(name, ".xls")) + { + return XL_FORMAT_XLS; + } + if (ends_with_ci(name, ".csv")) + { + return XL_FORMAT_CSV; + } + return XL_FORMAT_AUTO; } - return XL_FORMAT_AUTO; + } @@ -543,10 +507,6 @@ namespace xl public: RowView() = default; - RowView(std::span payload, size_t count) : payload_(payload), count_(count) {} - - RowView(const xl_row_cell *cells, size_t count) : cells_(cells), count_(count) {} - size_t size() const { return count_; } bool empty() const { return count_ == 0; } @@ -554,18 +514,12 @@ namespace xl /// @param index Position among the row's cells, below size(). CellView operator[](size_t index) const { - if (cells_ != nullptr) - { - const xl_row_cell &raw = cells_[index]; - const size_t length = raw.value_len > 0 ? static_cast(raw.value_len) : 0; - return CellView{raw.column, static_cast(raw.type), - length == 0 || raw.value == nullptr - ? std::string_view{} - : std::string_view(reinterpret_cast(raw.value), length)}; - } - auto it = begin(); - std::advance(it, static_cast(index)); - return *it; + const xl_row_cell &raw = cells_[index]; + const size_t length = raw.value_len > 0 ? static_cast(raw.value_len) : 0; + return CellView{raw.column, static_cast(raw.type), + length == 0 || raw.value == nullptr + ? std::string_view{} + : std::string_view(reinterpret_cast(raw.value), length)}; } class iterator @@ -576,20 +530,12 @@ namespace xl using difference_type = ptrdiff_t; iterator() = default; - iterator(const RowView *row, size_t index) : row_(row), index_(index) - { - if (row_ != nullptr) - { - next_ = row_->payload_.data(); - } - load(); - } + iterator(const RowView *row, size_t index) : row_(row), index_(index) {} - CellView operator*() const { return current_; } + CellView operator*() const { return (*row_)[index_]; } iterator &operator++() { ++index_; - load(); return *this; } iterator operator++(int) @@ -601,59 +547,30 @@ namespace xl bool operator==(const iterator &other) const { return index_ == other.index_; } private: - void load() - { - if (row_ == nullptr || index_ >= row_->count_) - { - return; - } - if (row_->cells_ != nullptr) - { - current_ = (*row_)[index_]; - return; - } - const uint8_t *end = row_->payload_.data() + row_->payload_.size(); - int32_t header[3]; - if (static_cast(end - next_) < sizeof(header)) - { - index_ = row_->count_; - return; - } - std::memcpy(header, next_, sizeof(header)); - const uint8_t *value = next_ + sizeof(header); - if (header[2] < 0 || static_cast(end - value) < static_cast(header[2])) - { - index_ = row_->count_; - return; - } - current_ = CellView{header[0], static_cast(header[1]), - std::string_view(reinterpret_cast(value), static_cast(header[2]))}; - next_ = value + header[2]; - } - const RowView *row_{}; size_t index_{}; - const uint8_t *next_{}; - CellView current_{}; }; iterator begin() const { return iterator(this, 0); } iterator end() const { return iterator(this, count_); } private: - std::span payload_{}; + friend class RowCursor; + friend class DecodedRows; + friend struct detail::Access; + + RowView(const xl_row_cell *cells, size_t count) : cells_(cells), count_(count) {} + const xl_row_cell *cells_{}; size_t count_{}; }; - /// @brief Every row of a sheet, or every row a cursor had left, decoded in one native call. + /// @brief Every row of a sheet, decoded in one native call. /// /// Owns that memory until destruction; the RowView it hands out borrow from it. class DecodedRows { public: - explicit DecodedRows(xl_rows_decoded raw) : raw_(raw) {} - DecodedRows(const DecodedRows &) = delete; DecodedRows &operator=(const DecodedRows &) = delete; @@ -714,6 +631,10 @@ namespace xl iterator end() const { return iterator(this, size()); } private: + friend class Sheet; + + explicit DecodedRows(xl_rows_decoded raw) : raw_(raw) {} + void release() { if (raw_.rows != nullptr) @@ -734,10 +655,6 @@ namespace xl class RowCursor { public: - /// @brief Takes ownership of a cursor opened through the C ABI. - /// @param rows Cursor from `xl_rows_open`, closed by this object. - explicit RowCursor(xl_rows *rows) noexcept : rows_(rows) {} - RowCursor(const RowCursor &) = delete; RowCursor &operator=(const RowCursor &) = delete; @@ -772,23 +689,11 @@ namespace xl return RowView(row.cells, static_cast(row.cell_count)); } - /// @brief Every row this cursor has not yet returned, decoded in one native call. - std::expected read_all_decoded() - { - xl_rows_decoded raw{}; - const int32_t status = xl_rows_read_all_decoded(rows_, &raw); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return DecodedRows(raw); - } + private: + friend class Sheet; - /// @brief The native cursor, for calling the C ABI directly. - /// @return The cursor, still owned by this object. - xl_rows *handle() const noexcept { return rows_; } + explicit RowCursor(xl_rows *rows) noexcept : rows_(rows) {} - private: void close() noexcept { if (rows_ != nullptr) @@ -817,11 +722,6 @@ namespace xl class Sheet { public: - /// @brief A sheet of a workbook opened through the C ABI. Workbook hands these out. - /// @param handle Workbook handle, not owned. - /// @param index 0-based sheet position. - Sheet(xl_workbook *handle, int32_t index) noexcept : handle_(handle), index_(index) {} - /// @brief 0-based position in the workbook. int32_t index() const noexcept { return index_; } @@ -858,14 +758,20 @@ namespace xl } /// @brief The whole sheet from its first row, decoded in one native call. - std::expected read_all_decoded() const + std::expected read_all() const { auto cursor = rows(); if (!cursor.has_value()) { return std::unexpected(cursor.error()); } - return cursor->read_all_decoded(); + xl_rows_decoded raw{}; + const int32_t status = xl_rows_read_all_decoded(cursor->rows_, &raw); + if (status != XL_OK) + { + return std::unexpected(detail::make_error(status)); + } + return DecodedRows(raw); } /// @brief Guesses each column's name and type from a sample of the sheet. @@ -910,16 +816,28 @@ namespace xl return columns; } - /// @brief The native workbook handle. - /// @return The handle, not owned: it dies with its Workbook. - xl_workbook *handle() const noexcept { return handle_; } - private: + friend class Workbook; + friend struct detail::Access; + + Sheet(xl_workbook *handle, int32_t index) noexcept : handle_(handle), index_(index) {} + xl_workbook *handle_ = nullptr; int32_t index_ = 0; }; - /// @brief Random-access bytes for Workbook::open_source. + namespace detail + { + struct Access + { + template + static R make(Args &&...args) { return R(std::forward(args)...); } + + static xl_workbook *handle(const Sheet &sheet) noexcept { return sheet.handle_; } + }; + } + + /// @brief Random-access bytes for Workbook::open. /// /// `read_at` is called from several threads at once. class Source @@ -936,7 +854,7 @@ namespace xl virtual std::expected read_at(std::uint64_t offset, std::span buffer) const = 0; }; - /// @brief Sequential bytes for Workbook::open_stream. + /// @brief Sequential bytes for Workbook::open. /// /// `read` is never called concurrently. class InputStream @@ -1027,66 +945,33 @@ namespace xl /// @brief Opens the workbook at a path. /// @param path File to open. - /// @param format An `XL_FORMAT_*` constant. `XL_FORMAT_AUTO` detects XLSX, XLSB and XLS from the - /// file's signature; CSV has none and needs `XL_FORMAT_CSV`. - /// @param options Reader limits and CSV dialect, or null for the defaults. - /// @return The workbook. Fails if the native library's ABI does not match this header. - static std::expected open(std::string_view path, int32_t format = XL_FORMAT_AUTO, - const OpenOptions *options = nullptr) + /// @param options Format, reader limits and CSV dialect. + /// @return The workbook. Every overload fails if the native library's ABI does not match this + /// header. + static std::expected open(std::string_view path, const OpenOptions &options = {}) { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } - xl_workbook *handle = nullptr; - xl_open_options c_options{}; - const xl_open_options *c_options_ptr = detail::lower_options(options, c_options); - int32_t status = xl_open_file(reinterpret_cast(path.data()), - static_cast(path.size()), format, - c_options_ptr, &handle); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return Workbook(handle); + return open_with(options, [&](const xl_open_options *raw, xl_workbook **out) + { return xl_open_file(reinterpret_cast(path.data()), + static_cast(path.size()), options.format, raw, out); }); } /// @brief Opens a workbook from bytes in memory. /// @param data The file's bytes. The native library copies them, so they need not outlive the call. - /// @param format As for Workbook::open. - /// @param options As for Workbook::open. - static std::expected open_memory(std::span data, int32_t format = XL_FORMAT_AUTO, - const OpenOptions *options = nullptr) + /// @param options As for the path overload. + static std::expected open(std::span data, const OpenOptions &options = {}) { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } - xl_workbook *handle = nullptr; - xl_open_options c_options{}; - const xl_open_options *c_options_ptr = detail::lower_options(options, c_options); - int32_t status = xl_open_memory(data.data(), static_cast(data.size()), format, - c_options_ptr, &handle); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return Workbook(handle); + return open_with(options, [&](const xl_open_options *raw, xl_workbook **out) + { return xl_open_memory(data.data(), static_cast(data.size()), options.format, + raw, out); }); } /// @brief Opens a workbook over random-access bytes. /// @param source Owned by the library from here on. Read from several threads, so every sheet can /// still be read in parallel. - /// @param format As for Workbook::open. - /// @param options As for Workbook::open; `source_block_size` and `source_cache_bytes` tune the + /// @param options As for the path overload; `source_block_size` and `source_cache_bytes` tune the /// block cache. - static std::expected open_source(std::unique_ptr source, int32_t format = XL_FORMAT_AUTO, - const OpenOptions *options = nullptr) + static std::expected open(std::unique_ptr source, const OpenOptions &options = {}) { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } if (source == nullptr) { return std::unexpected(Error{XL_INVALID_ARGUMENT, "source is null"}); @@ -1096,20 +981,15 @@ namespace xl { return std::unexpected(Error{XL_INVALID_ARGUMENT, "source size exceeds INT64_MAX"}); } - xl_source raw{}; - raw.struct_size = sizeof(xl_source); - raw.length = static_cast(size); - raw.read_at = &detail::source_read_at; - raw.release = &detail::source_release; - raw.user_data = source.release(); - xl_workbook *handle = nullptr; - xl_open_options c_options{}; - int32_t status = xl_open_source(&raw, format, detail::lower_options(options, c_options), &handle); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return Workbook(handle); + return open_with(options, [&](const xl_open_options *raw, xl_workbook **out) + { + xl_source c_source{}; + c_source.struct_size = sizeof(xl_source); + c_source.length = static_cast(size); + c_source.read_at = &detail::source_read_at; + c_source.release = &detail::source_release; + c_source.user_data = source.release(); + return xl_open_source(&c_source, options.format, raw, out); }); } /// @brief Opens a workbook over a stream that cannot seek. @@ -1117,76 +997,22 @@ namespace xl /// A CSV is read as it arrives, and again from the start until 16 MiB of it were read; an XLSX, /// XLSB or XLS is read whole first. /// @param stream Owned by the library from here on. - /// @param format As for Workbook::open. - /// @param options As for Workbook::open; `max_buffered_bytes` caps what an XLSX, XLSB or XLS may - /// buffer. - static std::expected open_stream(std::unique_ptr stream, int32_t format = XL_FORMAT_AUTO, - const OpenOptions *options = nullptr) + /// @param options As for the path overload; `max_buffered_bytes` caps what an XLSX, XLSB or XLS + /// may buffer. + static std::expected open(std::unique_ptr stream, const OpenOptions &options = {}) { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } if (stream == nullptr) { return std::unexpected(Error{XL_INVALID_ARGUMENT, "stream is null"}); } - xl_stream raw{}; - raw.struct_size = sizeof(xl_stream); - raw.read = &detail::stream_read; - raw.release = &detail::stream_release; - raw.user_data = stream.release(); - xl_workbook *handle = nullptr; - xl_open_options c_options{}; - int32_t status = xl_open_stream(&raw, format, detail::lower_options(options, c_options), &handle); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return Workbook(handle); - } - - - /// @brief Number of sheets. - std::expected sheet_count() const - { - int32_t count = 0; - int32_t status = xl_sheet_count(handle_, &count); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return count; - } - - /// @brief Name of the sheet at `index`. - /// @param index 0-based sheet position. - std::expected sheet_name_at(int32_t index) const - { - return detail::fill_string([this, index](uint8_t *buffer, int32_t capacity, int32_t *out_len) - { return xl_sheet_name_at(handle_, index, buffer, capacity, out_len); }); - } - - /// @brief Every sheet's name, in workbook order. - std::expected, Error> sheet_names() const - { - auto count = sheet_count(); - if (!count.has_value()) - { - return std::unexpected(count.error()); - } - std::vector names; - names.reserve(static_cast(*count > 0 ? *count : 0)); - for (int32_t i = 0; i < *count; ++i) - { - auto name = sheet_name_at(i); - if (!name.has_value()) - { - return std::unexpected(name.error()); - } - names.push_back(std::move(*name)); - } - return names; + return open_with(options, [&](const xl_open_options *raw, xl_workbook **out) + { + xl_stream c_stream{}; + c_stream.struct_size = sizeof(xl_stream); + c_stream.read = &detail::stream_read; + c_stream.release = &detail::stream_release; + c_stream.user_data = stream.release(); + return xl_open_stream(&c_stream, options.format, raw, out); }); } /// @brief The sheet at `index`. @@ -1199,16 +1025,17 @@ namespace xl /// @brief Every sheet, in workbook order. Opens nothing. std::expected, Error> sheets() const { - auto count = sheet_count(); - if (!count.has_value()) + int32_t count = 0; + int32_t status = xl_sheet_count(handle_, &count); + if (status != XL_OK) { - return std::unexpected(count.error()); + return std::unexpected(detail::make_error(status)); } std::vector result; - result.reserve(static_cast(*count > 0 ? *count : 0)); - for (int32_t i = 0; i < *count; ++i) + result.reserve(static_cast(count > 0 ? count : 0)); + for (int32_t i = 0; i < count; ++i) { - result.emplace_back(handle_, i); + result.push_back(Sheet(handle_, i)); } return result; } @@ -1246,13 +1073,26 @@ namespace xl return flag != 0; } - /// @brief The native handle, for calling the C ABI directly. - /// @return The handle, still owned by this object. - xl_workbook *handle() const noexcept { return handle_; } - private: explicit Workbook(xl_workbook *handle) noexcept : handle_(handle) {} + template + static std::expected open_with(const OpenOptions &options, Call &&call) + { + if (const auto &abi = detail::check_abi_version(); !abi.has_value()) + { + return std::unexpected(abi.error()); + } + const xl_open_options raw = detail::to_c(options); + xl_workbook *handle = nullptr; + const int32_t status = call(&raw, &handle); + if (status != XL_OK) + { + return std::unexpected(detail::make_error(status)); + } + return Workbook(handle); + } + void close() noexcept { if (handle_ != nullptr) @@ -1266,84 +1106,79 @@ namespace xl }; - /// @brief The `XL_T_*` column type `T` reads and writes as, in `value`. - /// - /// Defined for std::string and std::string_view, integers, floating point, bool, the std::chrono - /// date and time types, and std::optional of any of them. - /// @tparam T A field type. - template - struct XlType; - - template <> - struct XlType + namespace detail { - static constexpr int32_t value = XL_T_STRING; - }; - template <> - struct XlType - { - static constexpr int32_t value = XL_T_STRING; - }; + template + struct XlType; - template - requires std::is_integral_v - struct XlType - { - static constexpr int32_t value = XL_T_I64; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_STRING; + }; - template - requires std::is_floating_point_v - struct XlType - { - static constexpr int32_t value = XL_T_F64; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_STRING; + }; - template <> - struct XlType - { - static constexpr int32_t value = XL_T_BOOL; - }; + template + requires std::is_integral_v + struct XlType + { + static constexpr int32_t value = XL_T_I64; + }; - template <> - struct XlType - { - static constexpr int32_t value = XL_T_TIMESTAMP; - }; + template + requires std::is_floating_point_v + struct XlType + { + static constexpr int32_t value = XL_T_F64; + }; - template <> - struct XlType - { - static constexpr int32_t value = XL_T_DATE; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_BOOL; + }; - template <> - struct XlType - { - static constexpr int32_t value = XL_T_DATE; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_TIMESTAMP; + }; - template <> - struct XlType - { - static constexpr int32_t value = XL_T_TIME; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_DATE; + }; - template <> - struct XlType> - { - static constexpr int32_t value = XL_T_TIME; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_DATE; + }; - template - struct XlType> - { - static constexpr int32_t value = XlType::value; - }; + template <> + struct XlType + { + static constexpr int32_t value = XL_T_TIME; + }; - namespace detail - { + template <> + struct XlType> + { + static constexpr int32_t value = XL_T_TIME; + }; + + template + struct XlType> + { + static constexpr int32_t value = XlType::value; + }; template struct IsOptional : std::false_type @@ -1363,27 +1198,23 @@ namespace xl template using unwrap_optional_t = typename IsOptional::Inner; - } + template + struct FieldBinding + { + std::array column_names; + T Class::*member; + using FieldType = T; + }; + } - /// @brief A member bound to a column by header name. Built by make_field. - /// @tparam Class The row type. - /// @tparam T The member's type. - /// @tparam N How many header names the binding accepts. - template - struct FieldBinding - { - std::array column_names; - T Class::*member; - using FieldType = T; - }; /// @brief Binds a member to the column with one header. /// @param name Header text. /// @param member The member to fill or write. /// @return The binding, for ExcelMapper::get_bindings. template - constexpr FieldBinding make_field(const char *name, T Class::*member) + constexpr detail::FieldBinding make_field(const char *name, T Class::*member) { return {{name}, member}; } @@ -1393,9 +1224,9 @@ namespace xl /// @param member The member to fill or write. /// @return The binding, for ExcelMapper::get_bindings. template - constexpr FieldBinding make_field(const char *(&&names)[N], T Class::*member) + constexpr detail::FieldBinding make_field(const char *(&&names)[N], T Class::*member) { - FieldBinding result{}; + detail::FieldBinding result{}; for (std::size_t i = 0; i < N; ++i) { result.column_names[i] = names[i]; @@ -1535,7 +1366,27 @@ namespace xl return instance; } - } + template + int32_t with_specs(const Sheet &sheet, Call &&call) + { + static constexpr auto bindings = ExcelMapper::get_bindings(); + static constexpr size_t num_fields = std::tuple_size_v; + std::array, num_fields> name_lens_storage{}; + const std::array specs = + build_specs(bindings, std::make_index_sequence{}, name_lens_storage); + return call(Access::handle(sheet), sheet.index(), specs.data(), static_cast(num_fields)); + } + + inline std::optional check_batch_size(int64_t batch_size) + { + if (batch_size > 0) + { + return std::nullopt; + } + return Error{XL_INVALID_ARGUMENT, "batch_size must be positive; parse_sheet and parse_arrow read a whole sheet"}; + } + + } /// @brief The rows of a typed parse, held in native columnar memory. @@ -1671,23 +1522,11 @@ namespace xl return (*this)[row]; } - /// @brief Every row converted to `T`. - std::vector to_vector() const - { - std::vector result; - result.reserve(static_cast(size())); - for (T item : *this) - { - result.push_back(std::move(item)); - } - return result; - } - - /// @brief Takes ownership of a table produced by the C ABI. - /// @param table Table from `xl_parse_typed` or `xl_typed_reader_next`, freed by this object. - static TableView from_raw(xl_table table) { return TableView(table); } - private: + template + friend class TypedReader; + friend struct detail::Access; + explicit TableView(xl_table table) noexcept : table_(table) {} xl_table table_{}; @@ -1705,22 +1544,15 @@ namespace xl std::expected, Error> parse_sheet(Sheet sheet, int32_t header_row = 1, int32_t degree_of_parallelism = 1) { - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t num_fields = std::tuple_size_v; - std::array, num_fields> name_lens_storage{}; - std::array specs_array = - detail::build_specs(bindings, std::make_index_sequence{}, name_lens_storage); - std::span specs(specs_array); - xl_table table{}; - int32_t status = xl_parse_typed(sheet.handle(), sheet.index(), specs.data(), - static_cast(specs.size()), header_row, - degree_of_parallelism, &table); + const int32_t status = detail::with_specs( + sheet, [&](xl_workbook *handle, int32_t index, const xl_column_spec *specs, int32_t count) + { return xl_parse_typed(handle, index, specs, count, header_row, degree_of_parallelism, &table); }); if (status != XL_OK) { return std::unexpected(detail::make_error(status)); } - return TableView::from_raw(table); + return detail::Access::make>(table); } @@ -1748,7 +1580,7 @@ namespace xl return *this; } - ~TypedReader() { xl_typed_reader_close(reader_); } + ~TypedReader() { xl_typed_reader_close(reader_); } /// @brief Reads the next batch. /// @return The batch, or empty at the end of the sheet. @@ -1764,7 +1596,7 @@ namespace xl { return std::unexpected(detail::make_error(status)); } - return std::optional>(TableView::from_raw(table)); + return std::optional>(TableView(table)); } class iterator @@ -1828,11 +1660,9 @@ namespace xl iterator begin() { return iterator(this); } std::default_sentinel_t end() const noexcept { return {}; } - /// @brief Takes ownership of a reader opened through the C ABI. - /// @param reader Reader from `xl_typed_reader_open`, closed by this object. - static TypedReader from_raw(xl_typed_reader *reader) { return TypedReader(reader); } - private: + friend struct detail::Access; + explicit TypedReader(xl_typed_reader *reader) noexcept : reader_(reader) {} xl_typed_reader *reader_ = nullptr; @@ -1842,133 +1672,110 @@ namespace xl /// @tparam T Row type, mapped by `ExcelMapper`. /// @param sheet The sheet to read. /// @param header_row As for parse_sheet. - /// @param batch_size Rows per batch: 0 makes one batch of the whole sheet; negative is an error. + /// @param batch_size Rows per batch; must be positive. template std::expected, Error> typed_reader(Sheet sheet, int32_t header_row = 1, int64_t batch_size = 10000) { - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t num_fields = std::tuple_size_v; - std::array, num_fields> name_lens_storage{}; - std::array specs_array = - detail::build_specs(bindings, std::make_index_sequence{}, name_lens_storage); - + if (std::optional problem = detail::check_batch_size(batch_size)) + { + return std::unexpected(std::move(*problem)); + } xl_typed_reader *reader = nullptr; - int32_t status = xl_typed_reader_open(sheet.handle(), sheet.index(), specs_array.data(), - static_cast(specs_array.size()), header_row, - batch_size, &reader); + const int32_t status = detail::with_specs( + sheet, [&](xl_workbook *handle, int32_t index, const xl_column_spec *specs, int32_t count) + { return xl_typed_reader_open(handle, index, specs, count, header_row, batch_size, &reader); }); if (status != XL_OK) { return std::unexpected(detail::make_error(status)); } - return TypedReader::from_raw(reader); + return detail::Access::make>(reader); } - /// @brief Writes columns as one sheet, copying nothing. - /// @param path File to create. - /// @param format An `XL_FORMAT_*` constant. `XL_FORMAT_AUTO` is rejected: a file being created has - /// no signature to detect. - /// @param columns Columns of equal length, borrowed for the call. Either every column has a name, - /// written as a header row, or none does. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. - inline std::expected write_columns(std::string_view path, int32_t format, - std::span columns, - const WriteOptions *options = nullptr) + namespace detail { - const std::expected &abi = detail::check_abi_version(); - if (!abi.has_value()) - { - return std::unexpected(abi.error()); - } - std::expected row_count = detail::validate_write_columns(columns); - if (!row_count.has_value()) + + inline void take_buffer(xl_buffer &buffer, std::vector &out) { - return std::unexpected(std::move(row_count.error())); + struct BufferGuard + { + xl_buffer *buffer; + ~BufferGuard() { xl_free_buffer(buffer); } + } guard{&buffer}; + out.assign(buffer.data, buffer.data + buffer.len); } - const size_t count = columns.size(); - std::vector name_slots(count); - std::vector name_lens(count); - std::vector specs(count); - std::vector raw_columns(count); - for (size_t i = 0; i < count; ++i) + template + std::expected write_table(std::span columns, Call &&call) { - detail::fill_write_column(columns[i], name_slots[i], name_lens[i], specs[i], raw_columns[i]); - } + const std::expected &abi = check_abi_version(); + if (!abi.has_value()) + { + return std::unexpected(abi.error()); + } + std::expected row_count = validate_write_columns(columns); + if (!row_count.has_value()) + { + return std::unexpected(std::move(row_count.error())); + } - xl_table table{static_cast(count), *row_count, raw_columns.data()}; - xl_write_options raw_options{}; - const xl_write_options *options_pointer = detail::lower_options(options, raw_options); + const size_t count = columns.size(); + std::vector name_slots(count); + std::vector name_lens(count); + std::vector specs(count); + std::vector raw_columns(count); + for (size_t i = 0; i < count; ++i) + { + fill_write_column(columns[i], name_slots[i], name_lens[i], specs[i], raw_columns[i]); + } - const int32_t status = xl_write_typed(reinterpret_cast(path.data()), - static_cast(path.size()), format, specs.data(), - &table, options_pointer); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); + const xl_table table{static_cast(count), *row_count, raw_columns.data()}; + const int32_t status = call(specs.data(), &table); + if (status != XL_OK) + { + return std::unexpected(make_error(status)); + } + return {}; } - return {}; + } - /// @brief write_columns with the format taken from the file's extension. - /// @param path File to create; its extension picks the format. - /// @param columns As for the overload that takes a format. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. + /// @brief Writes columns as one sheet, copying nothing. + /// @param path File to create. + /// @param columns Columns of equal length, borrowed for the call. Either every column has a name, + /// written as a header row, or none does. + /// @param options Format, sheet name, CSV dialect and format toggles. `XL_FORMAT_AUTO` takes the + /// format from the path's extension. inline std::expected write_columns(std::string_view path, std::span columns, - const WriteOptions *options = nullptr) - { - return write_columns(path, format_from_path(path), columns, options); + const WriteOptions &options = {}) + { + const int32_t format = options.format == XL_FORMAT_AUTO ? detail::format_from_path(path) : options.format; + const xl_write_options raw = detail::to_c(options); + return detail::write_table(columns, [&](const xl_column_spec *specs, const xl_table *table) + { return xl_write_typed(reinterpret_cast(path.data()), + static_cast(path.size()), format, specs, table, + &raw); }); } /// @brief write_columns into memory. - /// @param format As for write_columns. There is no default: no file name to infer it from. - /// @param columns As for write_columns. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. - /// @return The file's bytes. - inline std::expected, Error> write_columns_to_memory(int32_t format, - std::span columns, - const WriteOptions *options = nullptr) - { - const std::expected &abi = detail::check_abi_version(); - if (!abi.has_value()) - { - return std::unexpected(abi.error()); - } - std::expected row_count = detail::validate_write_columns(columns); - if (!row_count.has_value()) - { - return std::unexpected(std::move(row_count.error())); - } - - const size_t count = columns.size(); - std::vector name_slots(count); - std::vector name_lens(count); - std::vector specs(count); - std::vector raw_columns(count); - for (size_t i = 0; i < count; ++i) - { - detail::fill_write_column(columns[i], name_slots[i], name_lens[i], specs[i], raw_columns[i]); - } - - xl_table table{static_cast(count), *row_count, raw_columns.data()}; - xl_write_options raw_options{}; - const xl_write_options *options_pointer = detail::lower_options(options, raw_options); - - xl_buffer buffer{}; - const int32_t status = xl_write_typed_to_memory(format, specs.data(), &table, options_pointer, &buffer); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - struct BufferGuard - { - xl_buffer *buffer; - ~BufferGuard() { xl_free_buffer(buffer); } - } guard{&buffer}; - return std::vector(buffer.data, buffer.data + buffer.len); + /// @param out Replaced with the file's bytes; untouched on failure. + /// @param columns As for the path overload. + /// @param options As for the path overload, except that `format` must name a format. + inline std::expected write_columns(std::vector &out, std::span columns, + const WriteOptions &options = {}) + { + const xl_write_options raw = detail::to_c(options); + return detail::write_table(columns, [&](const xl_column_spec *specs, const xl_table *table) + { + xl_buffer buffer{}; + const int32_t status = xl_write_typed_to_memory(options.format, specs, table, &raw, &buffer); + if (status == XL_OK) + { + detail::take_buffer(buffer, out); + } + return status; }); } - /// @brief Encrypts a finished XLSX or XLSB. /// /// The encryption parameters are fixed at Excel's own defaults; the result opens in Excel, and in @@ -2243,84 +2050,58 @@ namespace xl } - /// @brief Writes rows as one sheet. - /// @param path File to create. - /// @param format As for write_columns. - /// @param rows Input range of a row type with an ExcelMapper, the mapping parse_sheet reads with. - /// An empty std::optional field writes a blank cell. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. - template - std::expected write_sheet(std::string_view path, int32_t format, R &&rows, - const WriteOptions *options = nullptr) + namespace detail { - using T = std::remove_cvref_t>; - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t field_count = std::tuple_size_v; - static constexpr auto indices = std::make_index_sequence{}; - typename detail::BuildersFor>::type builders{}; - if constexpr (std::ranges::sized_range) - { - detail::reserve_all(builders, static_cast(std::ranges::size(rows)), indices); - } - for (const auto &row : rows) - { - detail::push_row(builders, row, bindings, indices); - } - if (detail::any_overflowed(builders, indices)) + template + std::expected write_rows(Destination &destination, R &&rows, const WriteOptions &options) { - return std::unexpected(Error{XL_INVALID_ARGUMENT, - "a string column exceeds 2 GiB, which int32 offsets cannot address."}); + using T = std::remove_cvref_t>; + static constexpr auto bindings = ExcelMapper::get_bindings(); + static constexpr size_t field_count = std::tuple_size_v; + static constexpr auto indices = std::make_index_sequence{}; + typename BuildersFor>::type builders{}; + if constexpr (std::ranges::sized_range) + { + reserve_all(builders, static_cast(std::ranges::size(rows)), indices); + } + for (const auto &row : rows) + { + push_row(builders, row, bindings, indices); + } + + if (any_overflowed(builders, indices)) + { + return std::unexpected(Error{XL_INVALID_ARGUMENT, + "a string column exceeds 2 GiB, which int32 offsets cannot address."}); + } + + // refs borrows pointers into builders, so both must stay in the scope of the write call. + const std::array refs = to_refs(builders, bindings, indices); + return write_columns(destination, refs, options); } - // refs borrows pointers into builders, so both must stay in the scope of the write call. - const std::array refs = detail::to_refs(builders, bindings, indices); - return write_columns(path, format, refs, options); } - /// @brief write_sheet with the format taken from the file's extension. - /// @param path File to create; its extension picks the format. - /// @param rows As for the overload that takes a format. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. + /// @brief Writes rows as one sheet. + /// @param path File to create. + /// @param rows Input range of a row type with an ExcelMapper, the mapping parse_sheet reads with. + /// An empty std::optional field writes a blank cell. + /// @param options As for write_columns. template - std::expected write_sheet(std::string_view path, R &&rows, - const WriteOptions *options = nullptr) + std::expected write_sheet(std::string_view path, R &&rows, const WriteOptions &options = {}) { - return write_sheet(path, format_from_path(path), std::forward(rows), options); + return detail::write_rows(path, std::forward(rows), options); } /// @brief write_sheet into memory. - /// @param format As for write_columns. There is no default: no file name to infer it from. - /// @param rows As for write_sheet. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. - /// @return The file's bytes. + /// @param out Replaced with the file's bytes; untouched on failure. + /// @param rows As for the path overload. + /// @param options As for the path overload, except that `format` must name a format. template - std::expected, Error> write_sheet_to_memory(int32_t format, R &&rows, - const WriteOptions *options = nullptr) + std::expected write_sheet(std::vector &out, R &&rows, const WriteOptions &options = {}) { - using T = std::remove_cvref_t>; - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t field_count = std::tuple_size_v; - static constexpr auto indices = std::make_index_sequence{}; - typename detail::BuildersFor>::type builders{}; - if constexpr (std::ranges::sized_range) - { - detail::reserve_all(builders, static_cast(std::ranges::size(rows)), indices); - } - for (const auto &row : rows) - { - detail::push_row(builders, row, bindings, indices); - } - - if (detail::any_overflowed(builders, indices)) - { - return std::unexpected(Error{XL_INVALID_ARGUMENT, - "a string column exceeds 2 GiB, which int32 offsets cannot address."}); - } - - // refs borrows pointers into builders, so both must stay in the scope of the write call. - const std::array refs = detail::to_refs(builders, bindings, indices); - return write_columns_to_memory(format, refs, options); + return detail::write_rows(out, std::forward(rows), options); } @@ -2328,69 +2109,46 @@ namespace xl /// /// Per sheet: start_sheet, then for each row start_row, one write per cell from left to right, and /// end_row; then end_sheet. A call out of order returns an error rather than corrupting the - /// output. Closed on destruction. + /// output. Finish with finish(); destruction finishes too, but cannot report a failure. class WriterHandle { public: WriterHandle(const WriterHandle &) = delete; WriterHandle &operator=(const WriterHandle &) = delete; - WriterHandle(WriterHandle &&other) noexcept : handle_(std::exchange(other.handle_, nullptr)) {} + WriterHandle(WriterHandle &&other) noexcept + : handle_(std::exchange(other.handle_, nullptr)), out_(std::exchange(other.out_, nullptr)) {} WriterHandle &operator=(WriterHandle &&other) noexcept { if (this != &other) { - close(); + (void)finish(); handle_ = std::exchange(other.handle_, nullptr); + out_ = std::exchange(other.out_, nullptr); } return *this; } - ~WriterHandle() { close(); } + ~WriterHandle() { (void)finish(); } /// @brief Creates a file to write into. /// @param path File to create. - /// @param format An `XL_FORMAT_*` constant other than `XL_FORMAT_AUTO`. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. - static std::expected open(std::string_view path, int32_t format, - const WriteOptions *options = nullptr) + /// @param options As for write_columns. + static std::expected open(std::string_view path, const WriteOptions &options = {}) { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } - xl_writer_handle *handle = nullptr; - xl_write_options c_options{}; - const xl_write_options *c_options_ptr = detail::lower_options(options, c_options); - int32_t status = xl_open_write_handle(reinterpret_cast(path.data()), - static_cast(path.size()), format, - c_options_ptr, &handle); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return WriterHandle(handle); + const int32_t format = options.format == XL_FORMAT_AUTO ? detail::format_from_path(path) : options.format; + return open_with(options, nullptr, [&](const xl_write_options *raw, xl_writer_handle **out) + { return xl_open_write_handle(reinterpret_cast(path.data()), + static_cast(path.size()), format, raw, out); }); } - /// @brief Writes into memory instead of a file; read the result with bytes(). - /// @param format An `XL_FORMAT_*` constant other than `XL_FORMAT_AUTO`. - /// @param options Sheet name, CSV dialect and format toggles, or null for the defaults. - static std::expected open_memory(int32_t format, - const WriteOptions *options = nullptr) + /// @brief Writes into memory instead of a file. + /// @param out Replaced with the file's bytes when the handle finishes. Must outlive the handle. + /// @param options As for write_columns, except that `format` must name a format. + static std::expected open(std::vector &out, const WriteOptions &options = {}) { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } - xl_writer_handle *handle = nullptr; - xl_write_options c_options{}; - const xl_write_options *c_options_ptr = detail::lower_options(options, c_options); - int32_t status = xl_open_write_handle_to_memory(format, c_options_ptr, &handle); - if (status != XL_OK) - { - return std::unexpected(detail::make_error(status)); - } - return WriterHandle(handle); + return open_with(options, &out, [&](const xl_write_options *raw, xl_writer_handle **handle) + { return xl_open_write_handle_to_memory(options.format, raw, handle); }); } /// @brief Starts a sheet. @@ -2411,7 +2169,8 @@ namespace xl std::expected end_sheet() { return status_result(xl_end_sheet(handle_)); } /// @brief Writes the next cell. - /// @param value Any type XlType covers; an empty std::optional writes a blank cell. + /// @param value A string, integer, floating-point, bool or std::chrono date or time value, as a + /// field of an ExcelMapper row may hold; an empty std::optional writes a blank cell. template std::expected write(const T &value) { @@ -2420,7 +2179,7 @@ namespace xl { if (!value.has_value()) { - return write_null(XlType::value); + return write_null(detail::XlType::value); } return write(*value); } @@ -2476,52 +2235,67 @@ namespace xl /// @param type The column's `XL_T_*` constant. std::expected write_null(int32_t type) { return status_result(xl_write_null(handle_, type)); } - /// @brief Everything written so far, for a handle from open_memory. + /// @brief Completes the workbook and closes the handle; a handle opened over a vector fills it. /// - /// Finishes the workbook's trailing structure but leaves the handle open. - /// @return The file's bytes. - std::expected, Error> bytes() + /// Every call after the first does nothing. + std::expected finish() { + if (handle_ == nullptr) + { + return {}; + } + xl_writer_handle *handle = std::exchange(handle_, nullptr); + std::vector *out = std::exchange(out_, nullptr); xl_buffer buffer{}; - const int32_t status = xl_write_handle_bytes(handle_, &buffer); - if (status != XL_OK) + std::expected result{}; + if (out != nullptr) { - return std::unexpected(detail::make_error(status)); + result = status_result(xl_write_handle_bytes(handle, &buffer)); } - struct BufferGuard + const int32_t closed = xl_close_write_handle(handle); + if (!result.has_value()) { - xl_buffer *buffer; - ~BufferGuard() { xl_free_buffer(buffer); } - } guard{&buffer}; - return std::vector(buffer.data, buffer.data + buffer.len); + return result; + } + if (out != nullptr) + { + detail::take_buffer(buffer, *out); + } + return status_result(closed); } - /// @brief The native handle, for calling the C ABI directly. - /// @return The handle, still owned by this object. - xl_writer_handle *handle() const noexcept { return handle_; } - private: - explicit WriterHandle(xl_writer_handle *handle) noexcept : handle_(handle) {} + WriterHandle(xl_writer_handle *handle, std::vector *out) noexcept : handle_(handle), out_(out) {} - static std::expected status_result(int32_t status) + template + static std::expected open_with(const WriteOptions &options, std::vector *out, + Call &&call) { + if (const auto &abi = detail::check_abi_version(); !abi.has_value()) + { + return std::unexpected(abi.error()); + } + const xl_write_options raw = detail::to_c(options); + xl_writer_handle *handle = nullptr; + const int32_t status = call(&raw, &handle); if (status != XL_OK) { return std::unexpected(detail::make_error(status)); } - return {}; + return WriterHandle(handle, out); } - void close() noexcept + static std::expected status_result(int32_t status) { - if (handle_ != nullptr) + if (status != XL_OK) { - xl_close_write_handle(handle_); - handle_ = nullptr; + return std::unexpected(detail::make_error(status)); } + return {}; } xl_writer_handle *handle_ = nullptr; + std::vector *out_ = nullptr; }; @@ -2561,22 +2335,22 @@ namespace xl /// What the four shims reach through `user_data`. The C ABI shares one `user_data` across /// every worker thread and requires it to be read-only or internally synchronized, so the - /// seed callable is held by pointer-to-const and the "a callback threw" flag is atomic. - template + /// seed is held by pointer-to-const and the "a callback threw" flag is atomic. + template struct CsvContext { - const Seed *seed; + const Acc *seed; std::atomic threw{false}; }; - template + template int32_t csv_seed(void **out_state, void *user_data) noexcept { - auto *context = static_cast *>(user_data); + auto *context = static_cast *>(user_data); *out_state = nullptr; try { - *out_state = new Acc((*context->seed)()); + *out_state = new Acc(*context->seed); return XL_OK; } catch (...) @@ -2586,14 +2360,14 @@ namespace xl } } - template + template int32_t csv_accumulate(void *state, const xl_row *row, void *user_data) noexcept { - auto *context = static_cast *>(user_data); + auto *context = static_cast *>(user_data); try { const size_t count = row->cell_count > 0 ? static_cast(row->cell_count) : 0; - return static_cast(state)->accumulate(RowView(row->cells, count)); + return static_cast(state)->accumulate(Access::make(row->cells, count)); } catch (...) { @@ -2602,10 +2376,10 @@ namespace xl } } - template + template int32_t csv_combine(void *acc, void *next, void *user_data) noexcept { - auto *context = static_cast *>(user_data); + auto *context = static_cast *>(user_data); try { return static_cast(acc)->combine(*static_cast(next)); @@ -2617,21 +2391,21 @@ namespace xl } } - template + template void csv_free_state(void *state, void *) noexcept { delete static_cast(state); } - template - xl_csv_aggregation csv_aggregation(CsvContext &context) noexcept + template + xl_csv_aggregation csv_aggregation(CsvContext &context) noexcept { xl_csv_aggregation agg{}; agg.struct_size = static_cast(sizeof(xl_csv_aggregation)); - agg.seed = &csv_seed; - agg.accumulate = &csv_accumulate; - agg.combine = &csv_combine; - agg.free_state = &csv_free_state; + agg.seed = &csv_seed; + agg.accumulate = &csv_accumulate; + agg.combine = &csv_combine; + agg.free_state = &csv_free_state; agg.user_data = &context; return agg; } @@ -2644,8 +2418,8 @@ namespace xl /// was later re-read and discarded would otherwise vanish. Keying on the flag rather than on /// the status is also what keeps `kCsvCallbackException` distinguishable from a caller that /// legitimately returned `INT32_MAX`. - template - std::expected csv_finish(const CsvContext &context, int32_t status, void *state) + template + std::expected csv_finish(const CsvContext &context, int32_t status, void *state) { const bool threw = context.threw.load(std::memory_order_relaxed); @@ -2680,100 +2454,66 @@ namespace xl } return std::move(*owned); } - } - /// @brief Folds the CSV file at `path` into one `Acc` across several threads. - /// - /// `seed` is called at least once per partition, on worker threads, and may be called again for - /// a partition that has to be re-read - a seeded accumulator can therefore be destroyed without - /// ever being combined. The call blocks until the run finishes, so `seed` may capture locals - /// freely. It must be const-callable: the C ABI shares one `user_data` across every worker and - /// requires it to be read-only or internally synchronized, so the shims invoke it through a - /// reference to const. A capturing `mutable` lambda therefore fails this constraint cleanly - /// instead of hard-erroring inside a shim. The `remove_cvref_t` is load-bearing - `Seed` deduces - /// to an lvalue reference for a named callable, and `const Seed &` would collapse the `const` - /// away. - /// @tparam Acc The accumulator; see CsvAccumulator. - /// @param path CSV file to read. - /// @param seed Makes a fresh `Acc` for a partition. - /// @param options Threads, header row and CSV dialect, or null for the defaults. - /// @return The combined accumulator. - template - requires std::invocable &> - std::expected aggregate_csv_file(std::string_view path, Seed &&seed, - const CsvParallelOptions *options = nullptr) - { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) + template + std::expected aggregate_with(const Acc &seed, const OpenOptions &options, int32_t header_row, + int32_t degree_of_parallelism, Call &&call) { - return std::unexpected(abi.error()); + if (const auto &abi = check_abi_version(); !abi.has_value()) + { + return std::unexpected(abi.error()); + } + CsvContext context{std::addressof(seed), {}}; + const xl_csv_aggregation agg = csv_aggregation(context); + const xl_csv_parallel_options raw = to_c(options, header_row, degree_of_parallelism); + void *state = nullptr; + const int32_t status = call(&agg, &raw, &state); + return csv_finish(context, status, state); } - - using SeedType = std::remove_cvref_t; - detail::CsvContext context{std::addressof(seed), {}}; - const xl_csv_aggregation agg = detail::csv_aggregation(context); - xl_csv_parallel_options c_options{}; - const xl_csv_parallel_options *c_options_ptr = detail::lower_options(options, c_options); - void *state = nullptr; - const int32_t status = xl_csv_aggregate_file( - reinterpret_cast(path.data()), static_cast(path.size()), - &agg, c_options_ptr, &state); - return detail::csv_finish(context, status, state); } - /// @brief Folds the CSV file at `path` into one `Acc`, seeding each partition with `Acc()`. + /// @brief Folds the CSV file at `path` into one `Acc` across several threads. + /// + /// Each partition starts from a copy of `seed`, made on a worker thread; a partition that has to + /// be re-read starts again from a fresh copy, so a copy can be destroyed without ever being + /// combined. Copies are made concurrently from the one `seed`, which the call only reads. /// @tparam Acc The accumulator; see CsvAccumulator. /// @param path CSV file to read. - /// @param options Threads, header row and CSV dialect, or null for the defaults. + /// @param seed The state every partition starts from. + /// @param options `csv_delimiter`, `csv_quote`, `csv_detect_bom` and `max_cell_bytes` apply; the + /// other fields are ignored. + /// @param header_row 1-based header row, not aggregated; 0 means none. + /// @param degree_of_parallelism Worker threads; 0 uses every processor, 1 runs sequentially. /// @return The combined accumulator. template - requires std::default_initializable - std::expected aggregate_csv_file(std::string_view path, - const CsvParallelOptions *options = nullptr) - { - auto seed = [] { return Acc(); }; - return aggregate_csv_file(path, seed, options); - } - - /// @brief Folds an in-memory CSV buffer into one `Acc`. - /// - /// Otherwise identical to aggregate_csv_file. - /// @tparam Acc The accumulator; see CsvAccumulator. - /// @param data The CSV bytes. Not copied, so they must stay valid for the call. - /// @param seed As for aggregate_csv_file. - /// @param options Threads, header row and CSV dialect, or null for the defaults. - /// @return The combined accumulator. - template - requires std::invocable &> - std::expected aggregate_csv_memory(std::span data, Seed &&seed, - const CsvParallelOptions *options = nullptr) - { - if (const auto &abi = detail::check_abi_version(); !abi.has_value()) - { - return std::unexpected(abi.error()); - } - - using SeedType = std::remove_cvref_t; - detail::CsvContext context{std::addressof(seed), {}}; - const xl_csv_aggregation agg = detail::csv_aggregation(context); - xl_csv_parallel_options c_options{}; - const xl_csv_parallel_options *c_options_ptr = detail::lower_options(options, c_options); - void *state = nullptr; - const int32_t status = xl_csv_aggregate_memory( - data.data(), static_cast(data.size()), &agg, c_options_ptr, &state); - return detail::csv_finish(context, status, state); + requires std::copy_constructible + std::expected aggregate_csv(std::string_view path, const Acc &seed = Acc{}, + const OpenOptions &options = {}, int32_t header_row = 0, + int32_t degree_of_parallelism = 0) + { + return detail::aggregate_with(seed, options, header_row, degree_of_parallelism, + [&](const xl_csv_aggregation *agg, const xl_csv_parallel_options *raw, void **state) + { return xl_csv_aggregate_file(reinterpret_cast(path.data()), + static_cast(path.size()), agg, raw, state); }); } - /// @brief Folds an in-memory CSV buffer into one `Acc`, seeding each partition with `Acc()`. + /// @brief Folds an in-memory CSV buffer into one `Acc`; otherwise as for the path overload. /// @tparam Acc The accumulator; see CsvAccumulator. /// @param data The CSV bytes. Not copied, so they must stay valid for the call. - /// @param options Threads, header row and CSV dialect, or null for the defaults. + /// @param seed As for the path overload. + /// @param options As for the path overload. + /// @param header_row As for the path overload. + /// @param degree_of_parallelism As for the path overload. /// @return The combined accumulator. template - requires std::default_initializable - std::expected aggregate_csv_memory(std::span data, - const CsvParallelOptions *options = nullptr) - { - auto seed = [] { return Acc(); }; - return aggregate_csv_memory(data, seed, options); + requires std::copy_constructible + std::expected aggregate_csv(std::span data, const Acc &seed = Acc{}, + const OpenOptions &options = {}, int32_t header_row = 0, + int32_t degree_of_parallelism = 0) + { + return detail::aggregate_with(seed, options, header_row, degree_of_parallelism, + [&](const xl_csv_aggregation *agg, const xl_csv_parallel_options *raw, void **state) + { return xl_csv_aggregate_memory(data.data(), static_cast(data.size()), + agg, raw, state); }); } } diff --git a/src/ExcelReader.Native/include/excelreader_arrow.hpp b/src/ExcelReader.Native/include/excelreader_arrow.hpp index 2a124184..f152396d 100644 --- a/src/ExcelReader.Native/include/excelreader_arrow.hpp +++ b/src/ExcelReader.Native/include/excelreader_arrow.hpp @@ -16,9 +16,11 @@ namespace xl { - /// @brief A sheet exported through the Arrow C Data Interface: one struct array and its schema. + /// @brief A sheet or one batch of it, exported through the Arrow C Data Interface: one struct + /// array and its schema. /// /// Releases both on destruction; move them out to hand them to an Arrow library. + /// ArrowStream::schema() returns one whose `array` is empty. struct ArrowTable { ArrowArray array{}; ///< One struct array holding every column. @@ -71,17 +73,11 @@ namespace xl std::expected parse_arrow(Sheet sheet, int32_t header_row = 1, int32_t degree_of_parallelism = 1) { - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t num_fields = std::tuple_size_v; - std::array, num_fields> name_lens_storage{}; - std::array specs_array = - detail::build_specs(bindings, std::make_index_sequence{}, name_lens_storage); - std::span specs(specs_array); - ArrowTable result; - int32_t status = xl_parse_arrow(sheet.handle(), sheet.index(), specs.data(), - static_cast(specs.size()), header_row, - degree_of_parallelism, &result.array, &result.schema); + const int32_t status = detail::with_specs( + sheet, [&](xl_workbook *handle, int32_t index, const xl_column_spec *specs, int32_t count) + { return xl_parse_arrow(handle, index, specs, count, header_row, degree_of_parallelism, + &result.array, &result.schema); }); if (status != XL_OK) { return std::unexpected(detail::make_error(status)); @@ -89,78 +85,6 @@ namespace xl return result; } - /// @brief Owns one ArrowSchema and releases it on destruction. - struct ArrowSchemaGuard - { - ArrowSchema schema{}; - - ArrowSchemaGuard() = default; - - ArrowSchemaGuard(const ArrowSchemaGuard &) = delete; - ArrowSchemaGuard &operator=(const ArrowSchemaGuard &) = delete; - - ArrowSchemaGuard(ArrowSchemaGuard &&other) noexcept - : schema(std::exchange(other.schema, ArrowSchema{})) - { - } - - ArrowSchemaGuard &operator=(ArrowSchemaGuard &&other) noexcept - { - if (this != &other) - { - release(); - schema = std::exchange(other.schema, ArrowSchema{}); - } - return *this; - } - - ~ArrowSchemaGuard() { release(); } - - void release() noexcept - { - if (schema.release != nullptr) - { - schema.release(&schema); - } - } - }; - - /// @brief Owns one ArrowArray and releases it on destruction. - struct ArrowArrayGuard - { - ArrowArray array{}; - - ArrowArrayGuard() = default; - - ArrowArrayGuard(const ArrowArrayGuard &) = delete; - ArrowArrayGuard &operator=(const ArrowArrayGuard &) = delete; - - ArrowArrayGuard(ArrowArrayGuard &&other) noexcept - : array(std::exchange(other.array, ArrowArray{})) - { - } - - ArrowArrayGuard &operator=(ArrowArrayGuard &&other) noexcept - { - if (this != &other) - { - release(); - array = std::exchange(other.array, ArrowArray{}); - } - return *this; - } - - ~ArrowArrayGuard() { release(); } - - void release() noexcept - { - if (array.release != nullptr) - { - array.release(&array); - } - } - }; - /// @brief typed_reader's Arrow counterpart: record batches pulled one at a time. /// /// Use from one thread at a time. @@ -187,48 +111,51 @@ namespace xl ~ArrowStream() { release(); } - /// @brief The schema every batch shares. - std::expected schema() + /// @brief The schema every batch shares, in a table whose `array` is empty. + std::expected schema() { if (stream_.release == nullptr) { return std::unexpected(Error{XL_ERROR, "the Arrow stream is released or moved-from"}); } - ArrowSchemaGuard guard; - int rc = stream_.get_schema(&stream_, &guard.schema); + ArrowTable table; + int rc = stream_.get_schema(&stream_, &table.schema); if (rc != 0) { return std::unexpected(stream_error(rc)); } - return guard; + return table; } /// @brief Reads the next batch. - /// @return The batch, or empty at the end of the sheet. - std::expected, Error> next() + /// @return The batch with its own copy of the schema, or empty at the end of the sheet. + std::expected, Error> next() { if (stream_.release == nullptr) { return std::unexpected(Error{XL_ERROR, "the Arrow stream is released or moved-from"}); } - ArrowArrayGuard guard; - int rc = stream_.get_next(&stream_, &guard.array); + ArrowTable table; + int rc = stream_.get_next(&stream_, &table.array); if (rc != 0) { return std::unexpected(stream_error(rc)); } - if (guard.array.release == nullptr) + if (table.array.release == nullptr) { - return std::optional{}; + return std::optional{}; } - return std::optional(std::move(guard)); + rc = stream_.get_schema(&stream_, &table.schema); + if (rc != 0) + { + return std::unexpected(stream_error(rc)); + } + return std::optional(std::move(table)); } - /// @brief Takes ownership of a stream produced by the C ABI. - /// @param stream Stream from `xl_parse_arrow_stream`, released by this object. - static ArrowStream from_raw(ArrowArrayStream stream) { return ArrowStream(stream); } - private: + friend struct detail::Access; + explicit ArrowStream(ArrowArrayStream stream) noexcept : stream_(stream) {} void release() noexcept @@ -262,21 +189,19 @@ namespace xl std::expected arrow_stream(Sheet sheet, int32_t header_row = 1, int64_t batch_size = 10000) { - static constexpr auto bindings = ExcelMapper::get_bindings(); - static constexpr size_t num_fields = std::tuple_size_v; - std::array, num_fields> name_lens_storage{}; - std::array specs_array = - detail::build_specs(bindings, std::make_index_sequence{}, name_lens_storage); - + if (std::optional problem = detail::check_batch_size(batch_size)) + { + return std::unexpected(std::move(*problem)); + } ArrowArrayStream stream{}; - int32_t status = xl_parse_arrow_stream(sheet.handle(), sheet.index(), specs_array.data(), - static_cast(specs_array.size()), header_row, - batch_size, &stream); + const int32_t status = detail::with_specs( + sheet, [&](xl_workbook *handle, int32_t index, const xl_column_spec *specs, int32_t count) + { return xl_parse_arrow_stream(handle, index, specs, count, header_row, batch_size, &stream); }); if (status != XL_OK) { return std::unexpected(detail::make_error(status)); } - return ArrowStream::from_raw(stream); + return detail::Access::make(stream); } } From 467d7d7e6e74855be2bef254853b0269b92d0d8d Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 17:18:29 -0300 Subject: [PATCH 10/21] docs: list the C++ API reductions in the v7 migration guide Co-Authored-By: Claude Opus 5.5 --- docs/guide/migrating-to-v7.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md index 73059d4c..52d5fef7 100644 --- a/docs/guide/migrating-to-v7.md +++ b/docs/guide/migrating-to-v7.md @@ -206,3 +206,26 @@ A sheet reads as rows (`rows()`), a typed table (`parse_typed()`, or `iter_parse | `writer.write_null(ColumnType.X)` | `(None, ColumnType.X)` as a value in `write_row` | | `OpenOptions(csv_max_cell_bytes=n)`, `OpenOptions(csv_intern_strings=b)` | `OpenOptions(max_cell_bytes=n)`, `OpenOptions(intern_strings=b)`, which now apply to CSV too | | `from excelreader import ArrowStream` | `from excelreader.arrow_stream import ArrowStream` | + +### C++: one overload set per operation + +`Workbook::open` takes a path, a byte span, a `Source` or an `InputStream`. `write_columns`, `write_sheet` and `WriterHandle::open` take a path or a `std::vector&`. Options are passed as `const Options& = {}`, and the format moved into `OpenOptions::format` and `WriteOptions::format` (`XL_FORMAT_AUTO` infers it from a path's extension). + +| v6 | v7 | +|---|---| +| `Workbook::open(path, format, &opts)` | `Workbook::open(path, {.format = format, ...})` | +| `Workbook::open_memory(span, ...)`, `open_source(...)`, `open_stream(...)` | `Workbook::open(span | source | stream, opts)` | +| `sheet_count()`, `sheet_name_at(i)`, `sheet_names()` | `sheets()`, `sheet(i).name()` | +| `Sheet::read_all_decoded()`, `RowCursor::read_all_decoded()` | `Sheet::read_all()` | +| `typed_reader` / `arrow_stream` with `batch_size = 0` | `parse_sheet` / `parse_arrow` | +| `TableView::to_vector()` | `std::vector(t.begin(), t.end())` | +| `handle()`, `from_raw`, constructors from C handles, `to_c()` | removed or private | +| `XlType`, `FieldBinding` | `xl::detail` | +| `OpenOptions::password(sv)` | `OpenOptions::password` field | +| `OpenOptions::csv_max_cell_bytes`, `csv_intern_strings` | `max_cell_bytes`, `intern_strings`, which now apply to CSV too | +| `write_columns_to_memory(format, cols, &opts)` | `write_columns(out_vector, cols, {.format = format})` | +| `write_sheet_to_memory(format, rows, &opts)` | `write_sheet(out_vector, rows, {.format = format})` | +| `format_from_path(path)` | internal | +| `WriterHandle::open_memory(format, &opts)` + `bytes()` | `WriterHandle::open(out_vector, {.format = format})` + `finish()` | +| `aggregate_csv_file` / `aggregate_csv_memory` with `CsvParallelOptions` | `aggregate_csv(path | span, seed, opts, header_row, dop)` | +| `ArrowSchemaGuard`, `ArrowArrayGuard` | `ArrowTable` | From bf29e32a06e52159151a6591b68ec943fd268bdf Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Wed, 14 Oct 2026 17:29:15 -0300 Subject: [PATCH 11/21] feat(rust)!: slim the public API for 7.0 Flatten the crate to one path per item at the root, hide the FFI, and fold duplicated entry points into option builders and enum arguments. - Raw bindings (Xl* structs, xl_* functions, XL_* constants) move to a private ffi module; derive support moves to #[doc(hidden)] __private. - Modules workbook, writer, writer_handle, rows, source, aggregate and arrow become private; their items are re-exported at the root. - Format, ColumnType and ErrorKind replace the XL_FORMAT_*, XL_T_* and status integers; Error becomes a struct with kind() and message(). - Opening: Workbook::open(path) plus OpenOptions::{open, open_bytes, open_source, open_reader}; format is an OpenOptions setting. Removed Workbook::{open_with, open_memory, open_source, open_reader, sheet_count, sheet_name_at, sheet_names}. - Reading: Sheet::read_all replaces read_all_decoded/DecodedRows and read_all_blob; Sheet::parse replaces parse_sheet/parse_sheet_parallel; Sheet::parse_arrow/parse_arrow_stream replace the arrow module's free functions; infer_schema takes a parse_text flag. - Writing: write_columns/write_sheet take a path and &WriteOptions; write_columns_to/write_sheet_to take an io::Write and replace the *_to_memory functions; format moves into WriteOptions (Auto infers from the extension). WriterHandle::{create, new, finish} replace open/open_with/open_memory/bytes; write_null is removed. - aggregate_csv(input, &OpenOptions, header_row, parallelism, seed) replaces aggregate_csv_file/aggregate_csv_memory/CsvParallelOptions. - OpenOptions/WriteOptions are builder-only; csv_max_cell_bytes and csv_intern_strings fold into max_cell_bytes/intern_strings. BREAKING CHANGE: every module path, the raw FFI surface, integer format and type codes, Error's enum shape and code(), and the removed functions listed above are gone; see the 7.0 migration guide. Co-Authored-By: Claude Opus 5.5 --- rust/excelreader-derive/src/lib.rs | 103 ++- rust/excelreader/README.md | 285 +++++---- rust/excelreader/benches/alloc_bench.rs | 10 +- rust/excelreader/benches/compare_bench.rs | 24 +- rust/excelreader/benches/parse_bench.rs | 12 +- rust/excelreader/benches/write_bench.rs | 62 +- rust/excelreader/src/aggregate.rs | 180 +++--- rust/excelreader/src/arrow.rs | 161 +++-- rust/excelreader/src/error.rs | 113 ++-- rust/excelreader/src/ffi.rs | 495 +++++++++++++++ rust/excelreader/src/lib.rs | 538 ++-------------- rust/excelreader/src/options.rs | 348 +++++++---- rust/excelreader/src/rows.rs | 134 ++-- rust/excelreader/src/source.rs | 79 ++- rust/excelreader/src/workbook.rs | 589 +++++++++--------- rust/excelreader/src/writer.rs | 286 ++++----- rust/excelreader/src/writer_handle.rs | 199 +++--- rust/excelreader/tests/aggregate.rs | 109 ++-- .../tests/derive_ui/fail/missing_name.rs | 2 +- .../tests/derive_ui/fail/non_struct.rs | 2 +- .../tests/derive_ui/fail/tuple_struct.rs | 2 +- .../derive_ui/fail/unknown_attribute_key.rs | 2 +- .../tests/derive_ui/fail/unsupported_type.rs | 2 +- .../excelreader/tests/derive_ui/pass/basic.rs | 12 +- rust/excelreader/tests/encrypted.rs | 89 +-- rust/excelreader/tests/parse_arrow.rs | 48 +- rust/excelreader/tests/parse_typed.rs | 103 +-- rust/excelreader/tests/rows.rs | 69 +- rust/excelreader/tests/sheets.rs | 46 +- rust/excelreader/tests/source.rs | 28 +- rust/excelreader/tests/typed_reader.rs | 11 +- rust/excelreader/tests/write_typed.rs | 352 +++++------ rust/excelreader/tests/writer_handle.rs | 124 ++-- 33 files changed, 2288 insertions(+), 2331 deletions(-) create mode 100644 rust/excelreader/src/ffi.rs diff --git a/rust/excelreader-derive/src/lib.rs b/rust/excelreader-derive/src/lib.rs index 5ef608ca..1cc46cac 100644 --- a/rust/excelreader-derive/src/lib.rs +++ b/rust/excelreader-derive/src/lib.rs @@ -1,5 +1,5 @@ //! `#[derive(ExcelMapper)]` for excelreader's `ExcelMapper` trait. Do not depend on this crate -//! directly - `excelreader` re-exports the macro from `excelreader::workbook::ExcelMapper` +//! directly - `excelreader` re-exports the macro as `excelreader::ExcelMapper` //! (same identifier as the trait; derive macros and traits occupy different namespaces, so //! there's no collision, same as `serde`/`serde_derive`). @@ -41,17 +41,17 @@ fn expand(input: DeriveInput) -> syn::Result { } Ok(quote! { - impl ::excelreader::workbook::ExcelMapper for #struct_name { - fn bindings() -> ::std::vec::Vec<::excelreader::workbook::ColumnBinding> { + impl ::excelreader::ExcelMapper for #struct_name { + fn bindings() -> ::std::vec::Vec<::excelreader::__private::ColumnBinding> { ::std::vec![ #(#bindings),* ] } } - impl ::excelreader::writer::ExcelWriter for #struct_name { + impl ::excelreader::ExcelWriter for #struct_name { fn to_columns( rows: &[Self], ) -> ::core::result::Result< - ::std::vec::Vec<::excelreader::writer::OwnedColumn>, + ::std::vec::Vec<::excelreader::__private::OwnedColumn>, ::excelreader::Error, > { let n = rows.len(); @@ -109,7 +109,7 @@ fn parse_field(field: &Field) -> syn::Result> { fn field_binding(parsed: &ParsedField<'_>) -> proc_macro2::TokenStream { let field_ident = parsed.ident; let name = &parsed.names; - let xl_type = parsed.kind.xl_type_tokens(); + let column_type = parsed.kind.column_type_tokens(); let value = parsed.kind.value_tokens(); let assign_value = if parsed.is_option { quote! { ::std::option::Option::Some(#value) } @@ -118,9 +118,9 @@ fn field_binding(parsed: &ParsedField<'_>) -> proc_macro2::TokenStream { }; quote! { - ::excelreader::workbook::ColumnBinding { + ::excelreader::__private::ColumnBinding { names: &[#(#name),*], - xl_type: #xl_type, + column_type: #column_type, assign: |r, col, row| r.#field_ident = #assign_value, } } @@ -262,7 +262,7 @@ impl WriteField { quote! { match &r.#ident { ::core::option::Option::Some(value) => { - ::excelreader::writer::set_valid(&mut #validity, row); + ::excelreader::__private::set_valid(&mut #validity, row); #append } ::core::option::Option::None => { #placeholder } @@ -277,12 +277,11 @@ impl WriteField { let field = self.name.value(); match self.kind { FieldKind::Str => quote! { - ::excelreader::writer::push_str(&mut #offsets, &mut #data, value.as_str())?; + ::excelreader::__private::push_str(&mut #offsets, &mut #data, value.as_str())?; }, FieldKind::Int => quote! { #values.push( - i64::try_from(*value).map_err(|_| ::excelreader::Error::from_status( - ::excelreader::XL_INVALID_ARGUMENT, + i64::try_from(*value).map_err(|_| ::excelreader::__private::invalid_argument( ::std::format!( "field `{}` holds a value that does not fit an i64 column", #field @@ -315,7 +314,7 @@ impl WriteField { fn placeholder_tokens(&self) -> proc_macro2::TokenStream { let (values, offsets, data) = (&self.values, &self.offsets, &self.data); if matches!(self.kind, FieldKind::Str) { - return quote! { ::excelreader::writer::push_str(&mut #offsets, &mut #data, "")?; }; + return quote! { ::excelreader::__private::push_str(&mut #offsets, &mut #data, "")?; }; } quote! { #values.push(::core::default::Default::default()); } } @@ -326,15 +325,15 @@ impl WriteField { let name = &self.name; let payload = match self.kind { FieldKind::Str => quote! { - ::excelreader::writer::OwnedColumnData::Str { offsets: #offsets, data: #data } + ::excelreader::__private::OwnedColumnData::Str { offsets: #offsets, data: #data } }, - FieldKind::Int => quote! { ::excelreader::writer::OwnedColumnData::I64(#values) }, - FieldKind::Float => quote! { ::excelreader::writer::OwnedColumnData::F64(#values) }, - FieldKind::Bool => quote! { ::excelreader::writer::OwnedColumnData::Bool(#values) }, - FieldKind::Date => quote! { ::excelreader::writer::OwnedColumnData::Date(#values) }, - FieldKind::Time => quote! { ::excelreader::writer::OwnedColumnData::Time(#values) }, + FieldKind::Int => quote! { ::excelreader::__private::OwnedColumnData::I64(#values) }, + FieldKind::Float => quote! { ::excelreader::__private::OwnedColumnData::F64(#values) }, + FieldKind::Bool => quote! { ::excelreader::__private::OwnedColumnData::Bool(#values) }, + FieldKind::Date => quote! { ::excelreader::__private::OwnedColumnData::Date(#values) }, + FieldKind::Time => quote! { ::excelreader::__private::OwnedColumnData::Time(#values) }, FieldKind::Timestamp => { - quote! { ::excelreader::writer::OwnedColumnData::Timestamp(#values) } + quote! { ::excelreader::__private::OwnedColumnData::Timestamp(#values) } } }; let validity_expr = if self.is_option { @@ -343,7 +342,7 @@ impl WriteField { quote! { ::core::option::Option::None } }; quote! { - ::excelreader::writer::OwnedColumn { + ::excelreader::__private::OwnedColumn { name: ::core::option::Option::Some(#name), data: #payload, validity: #validity_expr, @@ -387,35 +386,35 @@ impl FieldKind { } } - fn xl_type_tokens(&self) -> proc_macro2::TokenStream { + fn column_type_tokens(&self) -> proc_macro2::TokenStream { match self { - FieldKind::Str => quote! { ::excelreader::XL_T_STRING }, - FieldKind::Int => quote! { ::excelreader::XL_T_I64 }, - FieldKind::Float => quote! { ::excelreader::XL_T_F64 }, - FieldKind::Bool => quote! { ::excelreader::XL_T_BOOL }, - FieldKind::Date => quote! { ::excelreader::XL_T_DATE }, - FieldKind::Time => quote! { ::excelreader::XL_T_TIME }, - FieldKind::Timestamp => quote! { ::excelreader::XL_T_TIMESTAMP }, + FieldKind::Str => quote! { ::excelreader::ColumnType::String }, + FieldKind::Int => quote! { ::excelreader::ColumnType::I64 }, + FieldKind::Float => quote! { ::excelreader::ColumnType::F64 }, + FieldKind::Bool => quote! { ::excelreader::ColumnType::Bool }, + FieldKind::Date => quote! { ::excelreader::ColumnType::Date }, + FieldKind::Time => quote! { ::excelreader::ColumnType::Time }, + FieldKind::Timestamp => quote! { ::excelreader::ColumnType::Timestamp }, } } fn value_tokens(&self) -> proc_macro2::TokenStream { match self { - FieldKind::Str => quote! { ::excelreader::workbook::column_str(col, row).to_string() }, + FieldKind::Str => quote! { ::excelreader::__private::column_str(col, row).to_string() }, FieldKind::Int => quote! { - ::core::convert::TryFrom::try_from(::excelreader::workbook::column_i64(col, row)) + ::core::convert::TryFrom::try_from(::excelreader::__private::column_i64(col, row)) .expect("value does not fit this field's integer type") }, - FieldKind::Float => quote! { ::excelreader::workbook::column_f64(col, row) as _ }, - FieldKind::Bool => quote! { ::excelreader::workbook::column_bool(col, row) }, + FieldKind::Float => quote! { ::excelreader::__private::column_f64(col, row) as _ }, + FieldKind::Bool => quote! { ::excelreader::__private::column_bool(col, row) }, FieldKind::Date => { - quote! { ::core::convert::Into::into(::excelreader::workbook::column_date(col, row)) } + quote! { ::core::convert::Into::into(::excelreader::__private::column_date(col, row)) } } FieldKind::Time => { - quote! { ::core::convert::Into::into(::excelreader::workbook::column_time(col, row)) } + quote! { ::core::convert::Into::into(::excelreader::__private::column_time(col, row)) } } FieldKind::Timestamp => { - quote! { ::core::convert::Into::into(::excelreader::workbook::column_timestamp(col, row)) } + quote! { ::core::convert::Into::into(::excelreader::__private::column_timestamp(col, row)) } } } } @@ -459,11 +458,11 @@ mod tests { ) .expect("expand must succeed"); - assert!(output.contains("impl :: excelreader :: workbook :: ExcelMapper for Row")); - assert!(output.contains("XL_T_STRING")); - assert!(output.contains("XL_T_I64")); - assert!(output.contains("XL_T_F64")); - assert!(output.contains("XL_T_BOOL")); + assert!(output.contains("impl :: excelreader :: ExcelMapper for Row")); + assert!(output.contains("ColumnType :: String")); + assert!(output.contains("ColumnType :: I64")); + assert!(output.contains("ColumnType :: F64")); + assert!(output.contains("ColumnType :: Bool")); assert!(output.contains("\"Nome\"")); assert!(output.contains("\"Idade\"")); assert!(output.contains("\"Peso\"")); @@ -483,7 +482,7 @@ mod tests { .expect("expand must succeed"); assert!(output.contains("Option :: Some")); - assert!(output.contains("XL_T_BOOL")); + assert!(output.contains("ColumnType :: Bool")); } #[test] @@ -525,7 +524,7 @@ mod tests { )) .unwrap_or_else(|e| panic!("{ty} must be supported: {e}")); - assert!(output.contains("XL_T_I64"), "{ty} must use XL_T_I64"); + assert!(output.contains("ColumnType :: I64"), "{ty} must use an I64 column"); assert!(output.contains("try_from"), "{ty} must convert via TryFrom"); } } @@ -537,19 +536,19 @@ mod tests { r#"struct Row {{ #[excel(name = "V")] v: {ty}, }}"# )) .unwrap_or_else(|e| panic!("{ty} must be supported: {e}")); - assert!(output.contains("XL_T_F64"), "{ty} must use XL_T_F64"); + assert!(output.contains("ColumnType :: F64"), "{ty} must use an F64 column"); } } #[test] fn maps_temporal_types_onto_their_own_column_types() { for (ty, expected) in [ - ("Date", "XL_T_DATE"), - ("NaiveDate", "XL_T_DATE"), - ("Time", "XL_T_TIME"), - ("NaiveTime", "XL_T_TIME"), - ("Timestamp", "XL_T_TIMESTAMP"), - ("NaiveDateTime", "XL_T_TIMESTAMP"), + ("Date", "ColumnType :: Date"), + ("NaiveDate", "ColumnType :: Date"), + ("Time", "ColumnType :: Time"), + ("NaiveTime", "ColumnType :: Time"), + ("Timestamp", "ColumnType :: Timestamp"), + ("NaiveDateTime", "ColumnType :: Timestamp"), ] { let output = expand_str(&format!( r#"struct Row {{ #[excel(name = "D")] d: {ty}, }}"# @@ -571,7 +570,7 @@ mod tests { ) .expect("a qualified path must resolve by its last segment"); - assert!(output.contains("XL_T_DATE")); + assert!(output.contains("ColumnType :: Date")); } #[test] @@ -621,7 +620,7 @@ mod tests { ) .expect("expand must succeed"); - assert!(output.contains("impl :: excelreader :: writer :: ExcelWriter for Row")); + assert!(output.contains("impl :: excelreader :: ExcelWriter for Row")); assert!( output.contains("push_str"), "a String field must go through push_str" diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 59549d99..6cb14e56 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -2,19 +2,20 @@ # excelreader (Rust) -Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or -memory, with the full open-options surface), sheet navigation, schema inference, schema-driven typed -parse, schema-driven writing, row-by-row decoded reads, and Arrow export. +Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path, +memory, your own random-access source or any `Read`), sheet navigation, schema inference, +schema-driven typed parse, schema-driven writing, row-by-row reads, parallel CSV aggregation, and +Arrow export. Every public item lives at the crate root. ## Usage ```toml [dependencies] -excelreader = "2.1" +excelreader = "7.0" ``` ```rust,no_run -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::{ExcelMapper, Workbook}; #[derive(Default, ExcelMapper)] struct Row { @@ -25,24 +26,27 @@ struct Row { } let workbook = Workbook::open("book.xlsx")?; -let table = parse_sheet::(workbook.sheet(0)?, 1)?; +let table = workbook.sheet(0)?.parse::(1, 1)?; for row in table.iter() { /* ... */ } # Ok::<(), excelreader::Error>(()) ``` -`parse_sheet` takes a `Sheet`, which `Workbook::sheet` hands out (see below). +`Sheet::parse(header_row, degree_of_parallelism)` converts on the native side and never formats a +cell to text, so it is the fastest way to read a sheet whose columns you know. `header_row` is +1-based (`0` = no header); `degree_of_parallelism` is `0` for every processor, `1` sequential, `n` +up to `n` threads - XLSX, XLSB and CSV are split, and the table is the same whatever it is. ### Field types | Field type | Column | |---|---| -| `String` | `XL_T_STRING` | -| `i8`..`i64`, `isize`, `u8`..`u64`, `usize` | `XL_T_I64` | -| `f32`, `f64` | `XL_T_F64` | -| `bool` | `XL_T_BOOL` | -| `Date` / `chrono::NaiveDate` | `XL_T_DATE` | -| `Time` / `chrono::NaiveTime` | `XL_T_TIME` | -| `Timestamp` / `chrono::NaiveDateTime` | `XL_T_TIMESTAMP` | +| `String` | `ColumnType::String` | +| `i8`..`i64`, `isize`, `u8`..`u64`, `usize` | `ColumnType::I64` | +| `f32`, `f64` | `ColumnType::F64` | +| `bool` | `ColumnType::Bool` | +| `Date` / `chrono::NaiveDate` | `ColumnType::Date` | +| `Time` / `chrono::NaiveTime` | `ColumnType::Time` | +| `Timestamp` / `chrono::NaiveDateTime` | `ColumnType::Timestamp` | `Option` of any of the above makes the field `None` when the cell is null; a non-`Option` field is left at its `Default` instead. @@ -52,114 +56,108 @@ wrapping silently. `Date`/`Time`/`Timestamp` are dependency-free newtypes over t representation; enable the `chrono` feature to use `chrono`'s calendar types directly: ```toml -excelreader = { version = "2.1", features = ["chrono"] } +excelreader = { version = "7.0", features = ["chrono"] } ``` -### Sheets and schema inference +### Opening + +`Workbook::open(path)` sniffs the format and uses every library default. Everything else goes +through `OpenOptions`, in the style of `std::fs::OpenOptions`: set what you need, then call `open`, +`open_bytes`, `open_source` or `open_reader`. ```rust,no_run -# use excelreader::workbook::Workbook; -let workbook = Workbook::open("book.xlsx")?; -for (index, name) in workbook.sheet_names()?.iter().enumerate() { - println!("{index}: {name}"); -} +use excelreader::{Format, OpenOptions}; -// Guess a schema from the header row plus a sample of the data, before committing to one. -for column in workbook.sheet(1)?.infer_schema(1, 100)? { - println!("{:?} -> type {}", column.name, column.column_type); -} +let options = OpenOptions::new().format(Format::Csv).csv_delimiter(b';' as i32); +let from_path = options.open("data.csv")?; +let from_memory = options.open_bytes(b"a;b\n1;2\n")?; # Ok::<(), excelreader::Error>(()) ``` -`infer_schema_parse_text(header_row, sample_size)` also types text cells (every CSV field, or -numbers stored as text): integers, decimals, `true`/`false` and ISO dates. `parse_sheet_parallel` -and `arrow::parse_arrow_parallel` take a `degree_of_parallelism` (`0` = every processor, `1` = -sequential); XLSX, XLSB and CSV are split, and the result equals `parse_sheet`'s. +Format sniffing (`Format::Auto`, the default) does not detect CSV - set `Format::Csv`. The +resource limits (`max_total_decompressed_bytes`, `max_cell_bytes`, `max_shared_string_bytes`, +`max_zip_entries`, `max_buffered_bytes`) protect against hostile files; lower them when opening +untrusted input. + +### Sheets and schema inference A workbook hands out sheets. A `Sheet` is the workbook and an index: it holds no native resource, it is `Copy`, and every read on it opens its own cursor. ```rust,no_run -# use excelreader::workbook::Workbook; +# use excelreader::Workbook; let workbook = Workbook::open("report.xlsx")?; for sheet in workbook.sheets()? { println!("{} {:?}", sheet.name()?, sheet.visibility()?); } +// Guess a schema from the header row plus a sample of the data, before committing to one. let sheet = workbook.sheet_by_name("Totals")?.expect("the workbook has a Totals sheet"); -let mut rows = sheet.rows()?; -while let Some(row) = rows.next_row() { - let row = row?; - // ... +for column in sheet.infer_schema(1, 100, false)? { + println!("{:?} -> {:?}", column.name, column.column_type); } # Ok::<(), excelreader::Error>(()) ``` +`infer_schema`'s last argument, `parse_text`, also types text cells (every CSV field, or numbers +stored as text): integers, decimals, `true`/`false` and ISO dates. + `Workbook` is `Send + Sync`, so sheets can be read in parallel over one open workbook, which loads the shared-string table once: ```rust,no_run -# use excelreader::workbook::Workbook; +# use excelreader::Workbook; # let workbook = Workbook::open("report.xlsx")?; let tables = std::thread::scope(|scope| { let handles: Vec<_> = workbook .sheets()? .into_iter() - .map(|sheet| scope.spawn(move || sheet.read_all_blob())) + .map(|sheet| scope.spawn(move || sheet.read_all())) .collect(); handles.into_iter().map(|handle| handle.join().expect("no panic")).collect::, _>>() })?; # Ok::<(), excelreader::Error>(()) ``` -`Workbook::open_with` takes an explicit format and `OpenOptions`; `Workbook::open_memory` reads from -a byte slice. Note that format sniffing does not detect CSV - pass `XL_FORMAT_CSV` explicitly. - ### Encrypted workbooks -`OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm) through -`Workbook::open_with`/`open_memory`: +`OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm): ```rust,no_run -use excelreader::workbook::Workbook; -use excelreader::{Error, OpenOptions, XL_FORMAT_AUTO}; +use excelreader::{ErrorKind, OpenOptions}; -let options = OpenOptions::new().password("hunter2"); -match Workbook::open_with("protected.xlsx", XL_FORMAT_AUTO, Some(&options)) { +match OpenOptions::new().password("hunter2").open("protected.xlsx") { Ok(workbook) => { /* ... */ } - Err(Error::PasswordIncorrect { .. }) => { /* ask again */ } - Err(Error::PasswordRequired { .. }) => { /* no password was supplied */ } - Err(other) => { /* Error::Native - unsupported scheme or a corrupt file; not worth retrying */ } + Err(e) if e.kind() == ErrorKind::PasswordIncorrect => { /* ask again */ } + Err(e) if e.kind() == ErrorKind::PasswordRequired => { /* no password was supplied */ } + Err(other) => { /* unsupported scheme or a corrupt file; not worth retrying */ } } ``` -`Error::PasswordRequired`/`Error::PasswordIncorrect` are dedicated variants rather than the generic -error carrying `XL_STATUS_PASSWORD_REQUIRED`/`XL_STATUS_PASSWORD_INCORRECT`, so callers can -`match`/`matches!` on "needs a password" without hardcoding the status codes. Format sniffing does not -see through the CFB wrapper an encrypted file is stored in - pass `XL_FORMAT_AUTO` explicitly (as -above) rather than `XL_FORMAT_XLSX`, or the file is read as a plain ZIP and fails before the password -is ever checked. +Every `Error` carries an `ErrorKind` to branch on and the native detail in `message()`. Format +sniffing does not see through the CFB wrapper an encrypted file is stored in, so keep the default +`Format::Auto` rather than setting `Format::Xlsx`, or the file is read as a plain ZIP and fails +before the password is ever checked. Writing an encrypted workbook is a second step: write the plaintext package with `write_columns`/ `write_sheet`, then wrap it with `encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container - AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: ```rust,no_run -use excelreader::writer::{encrypt_package, write_columns}; -use excelreader::XL_FORMAT_XLSX; -# use excelreader::writer::{Column, ColumnData}; +use excelreader::{encrypt_package, write_columns, WriteOptions}; +# use excelreader::{Column, ColumnData}; # let ids = [1i64, 2, 3]; # let columns = [Column { name: Some("id"), data: ColumnData::I64(&ids), validity: None }]; -write_columns("plain.xlsx", XL_FORMAT_XLSX, &columns, None)?; +write_columns("plain.xlsx", &columns, &WriteOptions::new())?; encrypt_package("plain.xlsx", "secret.xlsx", "hunter2")?; # Ok::<(), excelreader::Error>(()) ``` `package_path` is read twice (it is not disposed or removed), so it must already be a finished file. Encryption parameters are fixed at Excel's own defaults - there are no options - and only XLSX/XLSB -packages can be encrypted, matching what `Workbook::open_with`/`open_memory` can decrypt. +packages can be encrypted, matching what `OpenOptions::password` can decrypt. ### Reading from your own bytes @@ -185,15 +183,15 @@ impl excelreader::Source for HttpSource { } } -let workbook = Workbook::open_source(source, XL_FORMAT_AUTO, None)?; +let workbook = OpenOptions::new().open_source(source)?; ``` Anything that implements `Read` works through `open_reader`. A CSV is read as it arrives; an XLSX, XLSB or XLS is read whole first. ```rust,no_run -# use excelreader::{workbook::Workbook, XL_FORMAT_CSV}; -let workbook = Workbook::open_reader(std::io::stdin(), XL_FORMAT_CSV, None)?; +# use excelreader::{Format, OpenOptions}; +let workbook = OpenOptions::new().format(Format::Csv).open_reader(std::io::stdin())?; # Ok::<(), excelreader::Error>(()) ``` @@ -203,19 +201,18 @@ needed the bytes. ### Arrow export (`arrow` feature) ```toml -excelreader = { version = "2.1", features = ["arrow"] } +excelreader = { version = "7.0", features = ["arrow"] } ``` ```rust,no_run # #[cfg(feature = "arrow")] # fn main() -> Result<(), excelreader::Error> { -use excelreader::arrow::parse_arrow; -use excelreader::workbook::Workbook; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::Workbook; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; -let batch = parse_arrow::(workbook.sheet(0)?, 1)?; +let batch = workbook.sheet(0)?.parse_arrow::(1, 1)?; println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); # Ok(()) # } @@ -225,20 +222,19 @@ println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); Off by default — arrow-rs is a large dependency and the typed-parse path needs none of it. -Batched: `parse_arrow_stream` is `parse_arrow` delivered a batch at a time, returning +Batched: `Sheet::parse_arrow_stream` is `parse_arrow` delivered a batch at a time, returning `ArrowChunks<'_>`, an `Iterator>` that also implements `RecordBatchReader`: ```rust,no_run # #[cfg(feature = "arrow")] # fn main() -> Result<(), excelreader::Error> { -use excelreader::arrow::parse_arrow_stream; -use excelreader::workbook::Workbook; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::Workbook; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; -let mut chunks = parse_arrow_stream::(workbook.sheet(0)?, 1, 10_000)?; +let mut chunks = workbook.sheet(0)?.parse_arrow_stream::(1, 10_000)?; while let Some(batch) = chunks.next() { let Ok(batch) = batch else { break }; // ArrowChunks does not fuse, unlike TypedChunks println!("{} rows", batch.num_rows()); @@ -259,13 +255,12 @@ first `Err` rather than continuing the loop. types in the table above apply unchanged: ```rust,no_run -use excelreader::writer::write_sheet; -use excelreader::XL_FORMAT_XLSX; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::{write_sheet, WriteOptions}; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } # let rows = vec![Row::default()]; -write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +write_sheet("out.xlsx", &rows, &WriteOptions::new())?; # Ok::<(), excelreader::Error>(()) ``` @@ -278,8 +273,7 @@ lifetimes on `Column<'a>` are what turn the ABI's borrow contract into something checks: ```rust,no_run -use excelreader::writer::{write_columns, Column, ColumnData}; -use excelreader::XL_FORMAT_XLSX; +use excelreader::{write_columns, Column, ColumnData, WriteOptions}; let ids = [1i64, 2, 3]; let columns = [Column { @@ -287,7 +281,7 @@ let columns = [Column { data: ColumnData::I64(&ids), validity: None, }]; -write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; +write_columns("out.xlsx", &columns, &WriteOptions::new())?; # Ok::<(), excelreader::Error>(()) ``` @@ -295,39 +289,53 @@ write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; length and reads `(rows + 7) / 8` bytes on trust, so a short slice would be a buffer overrun rather than a wrong answer. -`WriteOptions` sets the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB toggles. -`format_from_path` infers the format from an extension; it returns `XL_FORMAT_AUTO` for anything it -does not recognize, which the write then rejects - a file being created has no signature bytes to -sniff, so there is nothing to fall back on. +`WriteOptions` sets the format, the sheet name, the CSV dialect, and the XLS/XLSB and XLSX/XLSB +toggles. Its format defaults to `Format::Auto`, inferred from the path's extension; an extension it +does not recognize is an error - a file being created has no signature bytes to sniff. + +Each write has a path form and an `io::Write` form: `write_columns_to`/`write_sheet_to` build the +workbook in memory and write it to the writer in one `write_all`. A writer has no extension, so +these need an explicit format: + +```rust,no_run +use excelreader::{write_sheet_to, Format, WriteOptions}; +# #[derive(Default, excelreader::ExcelMapper)] +# struct Row { #[excel(name = "Name")] name: String } +# let rows = vec![Row::default()]; + +let mut bytes = Vec::new(); +write_sheet_to(&mut bytes, &rows, &WriteOptions::new().format(Format::Xlsx))?; +# Ok::<(), excelreader::Error>(()) +``` `write_sheet` walks the slice once and appends each field to its own buffer, monomorphized per field. That transpose is the only copy it makes; `write_columns` pays nothing. ### Streaming writes -`write_sheet`/`write_columns` build the whole table in memory first. `writer_handle::WriterHandle` is -the row-by-row alternative, writing directly as each call arrives instead: +`write_sheet`/`write_columns` build the whole table in memory first. `WriterHandle` is the +row-by-row alternative, writing directly as each call arrives instead: ```rust,no_run -use excelreader::writer_handle::WriterHandle; +use excelreader::{WriteOptions, WriterHandle}; -let mut handle = WriterHandle::open("out.xlsx", None)?; +let mut handle = WriterHandle::create("out.xlsx", &WriteOptions::new())?; handle.start_sheet("Summary")?; handle.start_row()?; handle.write_str(Some("Name"))?; handle.write_i64(Some(42))?; handle.end_row()?; handle.end_sheet()?; +handle.finish()?; # Ok::<(), excelreader::Error>(()) ``` -Call order mirrors the C ABI's `xl_writer_handle`: `open`/`open_with`/`open_memory`, then per sheet -`start_sheet..end_sheet`, each containing `start_row..end_row` with one `write_*` call per cell -(`None` writes a blank cell), left to right. A call out of order returns `Err` rather than -corrupting output. `open_memory` backs the handle with an in-memory buffer instead of a file; read -it out with `bytes()`. Dropping a `WriterHandle` closes and releases it, same as `Workbook` — call -`bytes()` (memory-backed) or reopen the path (file-backed) to observe the result rather than relying -on the drop for that. +Call order mirrors the C ABI's `xl_writer_handle`: `create` (a path) or `new` (any `io::Write`), +then per sheet `start_sheet..end_sheet`, each containing `start_row..end_row` with one `write_*` +call per cell (`None` writes a blank cell), left to right, then `finish`. A call out of order +returns `Err` rather than corrupting output. A handle from `new` writes the whole workbook to its +writer once, in `finish`. Dropping a handle without finishing it closes it on a best-effort basis, +discarding any error, and writes nothing to a writer. ## Reading rows one at a time @@ -335,7 +343,7 @@ on the drop for that. reuses, so iterating a sheet allocates nothing per row: ```rust,no_run -use excelreader::workbook::Workbook; +use excelreader::Workbook; let workbook = Workbook::open("book.xlsx")?; let mut cursor = workbook.sheet(0)?.rows()?; @@ -350,30 +358,27 @@ while let Some(row) = cursor.next_row() { ``` Because each row borrows the cursor's buffer, only one row is alive at a time — the borrow checker -enforces it. To hold every row at once, use `read_all_decoded`, which decodes the whole -sheet in one native call and keeps it alive until dropped: +enforces it. To hold every row at once, use `read_all`, which reads the whole sheet in one native +call into one buffer and keeps it alive until dropped: ```rust,no_run -# use excelreader::workbook::Workbook; +# use excelreader::Workbook; let workbook = Workbook::open("book.xlsx")?; -let rows = workbook.sheet(0)?.read_all_decoded()?; +let rows = workbook.sheet(0)?.read_all()?; for row in rows.iter() { println!("{} cells", row.len()); } # Ok::<(), excelreader::Error>(()) ``` -`parse_sheet` remains the fastest way to read a sheet whose columns you know — it converts on the -native side and never formats a cell to text. - ### Chunked reads -For a sheet too large to hold in memory at once, `Sheet::typed_chunks` is `parse_sheet` delivered +For a sheet too large to hold in memory at once, `Sheet::typed_chunks` is `Sheet::parse` delivered a batch at a time: ```rust,no_run -use excelreader::workbook::Workbook; -# #[derive(Default, excelreader::workbook::ExcelMapper)] +use excelreader::Workbook; +# #[derive(Default, excelreader::ExcelMapper)] # struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; @@ -387,32 +392,62 @@ for batch in chunks { `typed_chunks` returns `TypedChunks<'_, Row>`, an `Iterator, Error>>` that closes the native reader on drop. `batch_size` is rows per batch — `0` means one unbounded -batch, identical to `parse_sheet`; negative is an error. Chunked reads, of either kind (typed or +batch, identical to `parse`; negative is an error. Chunked reads, of either kind (typed or Arrow — see [Arrow export](#arrow-export-arrow-feature) above), are independent: several can be live on one workbook, and other reads do not disturb them. `TypedChunks` ends after yielding an `Err`, so a `for` loop stops there rather than spinning on it. +## Parallel CSV aggregation + +`aggregate_csv` folds a CSV file or buffer into one accumulator across several threads, without +building a table. The CSV dialect and cell limit come from `OpenOptions`: + +```rust,no_run +use excelreader::{aggregate_csv, CsvAccumulator, OpenOptions, RowRef}; + +struct Count(u64); +impl CsvAccumulator for Count { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + self.0 += 1; + Ok(()) + } + fn combine(&mut self, other: &mut Self) -> Result<(), i32> { + self.0 += other.0; + Ok(()) + } +} + +let options = OpenOptions::new().csv_delimiter(b';' as i32); +let rows = aggregate_csv("big.csv", &options, 1, 0, || Count(0))?; +# Ok::<(), excelreader::Error>(()) +``` + +The input is a path (`&str`, `&Path`, `&PathBuf`) or bytes (`&[u8]`); then `header_row` and +`degree_of_parallelism`, as in `Sheet::parse`. A nonzero code from a callback comes back as +`ErrorKind::Callback(code)`; a panic in one is resumed on the calling thread. + ## Bounds and panics -`TableView::get` returns `Option` and is `None` outside `0..len()`. The `column_*` accessors used -by generated bindings panic on an out-of-range row, a column type that does not match the binding, or -a string the native library returned as non-UTF-8 - each is a contract violation rather than -recoverable input. +`TableView::get` returns `Option` and is `None` outside `0..len()`. The accessors the derive's +generated code reads columns through panic on an out-of-range row, a column type that does not +match the binding, or a string the native library returned as non-UTF-8 - each is a contract +violation rather than recoverable input. ## Parallel aggregation on macOS -`aggregate_csv_file`/`aggregate_csv_memory` over a source large enough to partition (256 KB and up) -can crash the process on macOS arm64. The fault is inside the native runtime's signal handling, not -in this crate: a .NET thread resumes at address 0 after a signal the runtime sent it, which ends the -process with no message. It happens with any `degree_of_parallelism` above 1, and the same runtime -built into a plain C host does not show it, so a Rust host appears to make it far more likely. +`aggregate_csv` over a source large enough to partition (256 KB and up) can crash the process on +macOS arm64. The fault is inside the native runtime's signal handling, not in this crate: a .NET +thread resumes at address 0 after a signal the runtime sent it, which ends the process with no +message. It happens with any `degree_of_parallelism` above 1, and the same runtime built into a +plain C host does not show it, so a Rust host appears to make it far more likely. -`parse_sheet_parallel` and `arrow::parse_arrow_parallel` now fan out on XLSX and XLSB too; this has not been tested -on macOS, so treat them the same way. +`Sheet::parse` and `Sheet::parse_arrow` with a `degree_of_parallelism` above 1 fan out on XLSX and +XLSB too; this has not been tested on macOS, so treat them the same way. -Until it is fixed upstream, pass `degree_of_parallelism: 1` on macOS for a sequential run, or keep -sources under the 256 KB partitioning floor, where the library never fans out. Linux and Windows are -unaffected. The crate's own partitioned aggregation tests are `#[ignore]`d on macOS for this reason. +Until it is fixed upstream, pass a `degree_of_parallelism` of 1 on macOS for a sequential run, or +keep sources under the 256 KB partitioning floor, where the library never fans out. Linux and +Windows are unaffected. The crate's own partitioned aggregation tests are `#[ignore]`d on macOS for +this reason. ## Build notes @@ -421,9 +456,9 @@ matching GitHub Release. Set `EXCELREADER_NATIVE_LIB_DIR` to a directory contain copy instead (e.g. from `dotnet publish ../../src/ExcelReader.Native -r win-x64`) to skip that download - useful when building from source before a release exists yet. -Every constructor first checks the loaded library's `xl_abi_version()` against the `XL_ABI_VERSION` -this crate was compiled against, and fails with a explanatory error rather than reading native memory -through a layout that may have changed. +Every constructor first checks the loaded library's ABI revision against the one this crate was +compiled against, and fails with an explanatory error rather than reading native memory through a +layout that may have changed. ## Benchmarks @@ -431,7 +466,7 @@ Criterion suite in `benches/`. Measured on Windows 10 (22H2), 16 logical CPUs @ rustc 1.97.1 (Release), Criterion 0.8, 100 samples per benchmark (medians shown). `write_bench` takes 20 samples instead — each of its iterations writes a whole 65,535-row file. -`benches/parse_bench.rs` - `open`/`parse_sheet`/`infer_schema`, same methodology as the C++ suite: +`benches/parse_bench.rs` - `open`/`Sheet::parse`/`infer_schema`, same methodology as the C++ suite: | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| @@ -451,7 +486,7 @@ not the file's total row count. into an owned value (`String` for text) and fold it into one accumulator, so neither side gets a zero-copy advantage the other can't take: -| Format | ExcelReader (`parse_sheet::`) | calamine (`worksheet_range` + `Data` match) | +| Format | ExcelReader (`Sheet::parse::`) | calamine (`worksheet_range` + `Data` match) | |---|---:|---:| | XLSX | 79.1 ms | 279.7 ms | | XLSB | 46.9 ms | 87.5 ms | diff --git a/rust/excelreader/benches/alloc_bench.rs b/rust/excelreader/benches/alloc_bench.rs index 67eb3d37..be845ead 100644 --- a/rust/excelreader/benches/alloc_bench.rs +++ b/rust/excelreader/benches/alloc_bench.rs @@ -1,7 +1,7 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicU64, Ordering}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::{ExcelMapper, Workbook}; static BYTES_ALLOCATED: AtomicU64 = AtomicU64::new(0); @@ -112,7 +112,7 @@ fn main() { ITERATIONS, || Workbook::open(&small_path).expect("open must succeed"), |workbook| { - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook.sheet(0).expect("sheet 0").parse::(1, 1).expect("parse must succeed"); std::hint::black_box(table); }, ); @@ -122,7 +122,7 @@ fn main() { ITERATIONS, || Workbook::open(&small_path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 100).expect("infer_schema must succeed"); + let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 100, false).expect("infer_schema must succeed"); std::hint::black_box(schema); }, ); @@ -142,7 +142,7 @@ fn main() { ITERATIONS_LARGE, || Workbook::open(&large_path).expect("open must succeed"), |workbook| { - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + let table = workbook.sheet(0).expect("sheet 0").parse::(1, 1).expect("parse must succeed"); std::hint::black_box(table); }, ); @@ -152,7 +152,7 @@ fn main() { ITERATIONS_LARGE, || Workbook::open(&large_path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 1000).expect("infer_schema must succeed"); + let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 1000, false).expect("infer_schema must succeed"); std::hint::black_box(schema); }, ); diff --git a/rust/excelreader/benches/compare_bench.rs b/rust/excelreader/benches/compare_bench.rs index 5efb39e8..bed36249 100644 --- a/rust/excelreader/benches/compare_bench.rs +++ b/rust/excelreader/benches/compare_bench.rs @@ -1,8 +1,8 @@ use calamine::{open_workbook_auto, Data, Reader}; use criterion::{criterion_group, criterion_main, Criterion}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; -use excelreader::Date; +use excelreader::ExcelMapper; +use excelreader::{Date, Format, OpenOptions}; #[derive(Default, ExcelMapper)] struct FullRow { @@ -79,13 +79,13 @@ fn accumulate_data_cell(acc: &mut i64, cell: &Data) { } } -fn bench_excelreader(c: &mut Criterion, group: &str, path: String, format: i32) { +fn bench_excelreader(c: &mut Criterion, group: &str, path: String, format: Format) { c.bench_function(group, |b| { b.iter_batched( - || Workbook::open_with(&path, format, None).expect("open must succeed"), + || OpenOptions::new().format(format).open(&path).expect("open must succeed"), |workbook| { let table = - parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + workbook.sheet(0).expect("sheet 0").parse::(1, 1).expect("parse must succeed"); let mut acc = 0i64; for row in table.iter() { acc += accumulate_full_row(&row); @@ -97,13 +97,13 @@ fn bench_excelreader(c: &mut Criterion, group: &str, path: String, format: i32) }); } -fn bench_excelreader_parse_only(c: &mut Criterion, group: &str, path: String, format: i32) { +fn bench_excelreader_parse_only(c: &mut Criterion, group: &str, path: String, format: Format) { c.bench_function(group, |b| { b.iter_batched( - || Workbook::open_with(&path, format, None).expect("open must succeed"), + || OpenOptions::new().format(format).open(&path).expect("open must succeed"), |workbook| { let table = - parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + workbook.sheet(0).expect("sheet 0").parse::(1, 1).expect("parse must succeed"); std::hint::black_box(table.len()); std::hint::black_box(table); }, @@ -132,7 +132,7 @@ fn bench_calamine(c: &mut Criterion, group: &str, path: String) { } fn bench_excelreader_xlsx(c: &mut Criterion) { - bench_excelreader(c, "excelreader_xlsx_full", xlsx_path(), excelreader::XL_FORMAT_XLSX); + bench_excelreader(c, "excelreader_xlsx_full", xlsx_path(), Format::Xlsx); } fn bench_calamine_xlsx(c: &mut Criterion) { @@ -140,7 +140,7 @@ fn bench_calamine_xlsx(c: &mut Criterion) { } fn bench_excelreader_xlsb(c: &mut Criterion) { - bench_excelreader(c, "excelreader_xlsb_full", xlsb_path(), excelreader::XL_FORMAT_XLSB); + bench_excelreader(c, "excelreader_xlsb_full", xlsb_path(), Format::Xlsb); } fn bench_calamine_xlsb(c: &mut Criterion) { @@ -152,7 +152,7 @@ fn bench_excelreader_xlsx_parse_only(c: &mut Criterion) { c, "excelreader_xlsx_parse_only", xlsx_path(), - excelreader::XL_FORMAT_XLSX, + Format::Xlsx, ); } @@ -161,7 +161,7 @@ fn bench_excelreader_xlsb_parse_only(c: &mut Criterion) { c, "excelreader_xlsb_parse_only", xlsb_path(), - excelreader::XL_FORMAT_XLSB, + Format::Xlsb, ); } diff --git a/rust/excelreader/benches/parse_bench.rs b/rust/excelreader/benches/parse_bench.rs index 75f3ede1..a7de9d13 100644 --- a/rust/excelreader/benches/parse_bench.rs +++ b/rust/excelreader/benches/parse_bench.rs @@ -1,5 +1,5 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; +use excelreader::{ExcelMapper, Workbook}; #[derive(Default, ExcelMapper)] struct Row { @@ -54,7 +54,7 @@ fn bench_parse_sheet(c: &mut Criterion) { || Workbook::open(&path).expect("open must succeed"), |workbook| { let table = - parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("parse_sheet must succeed"); + workbook.sheet(0).expect("sheet 0").parse::(1, 1).expect("parse must succeed"); std::hint::black_box(table); }, criterion::BatchSize::SmallInput, @@ -68,7 +68,7 @@ fn bench_infer_schema(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 100) + let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 100, false) .expect("infer_schema must succeed"); std::hint::black_box(schema); }, @@ -93,8 +93,8 @@ fn bench_parse_sheet_large(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1) - .expect("parse_sheet must succeed"); + let table = workbook.sheet(0).expect("sheet 0").parse::(1, 1) + .expect("parse must succeed"); std::hint::black_box(table); }, criterion::BatchSize::SmallInput, @@ -108,7 +108,7 @@ fn bench_infer_schema_large(c: &mut Criterion) { b.iter_batched( || Workbook::open(&path).expect("open must succeed"), |workbook| { - let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 1000) + let schema = workbook.sheet(0).expect("sheet 0").infer_schema(1, 1000, false) .expect("infer_schema must succeed"); std::hint::black_box(schema); }, diff --git a/rust/excelreader/benches/write_bench.rs b/rust/excelreader/benches/write_bench.rs index f8c779bb..28b67b27 100644 --- a/rust/excelreader/benches/write_bench.rs +++ b/rust/excelreader/benches/write_bench.rs @@ -18,9 +18,9 @@ //! Machine: state the CPU, OS and toolchain version alongside any number published from this file. use criterion::{criterion_group, criterion_main, Criterion}; -use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; -use excelreader::writer::{write_columns, write_sheet, Column, OwnedColumn}; -use excelreader::{Date, XL_FORMAT_XLSX}; +use excelreader::{ + write_columns, write_sheet, Column, ColumnData, Date, ExcelMapper, Workbook, WriteOptions, +}; use std::path::{Path, PathBuf}; const FIXTURE: &str = "../../tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsx"; @@ -53,7 +53,11 @@ fn load_rows() -> Vec { present in the repo" ); let workbook = Workbook::open(FIXTURE).expect("the fixture must open"); - let table = parse_sheet::(workbook.sheet(0).expect("sheet 0"), 1).expect("the fixture must parse"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(1, 1) + .expect("the fixture must parse"); let rows: Vec = table.iter().collect(); assert!(!rows.is_empty(), "the fixture parsed to zero rows"); rows @@ -65,20 +69,57 @@ fn output_path(name: &str) -> PathBuf { path } +/// One string column's offsets and concatenated bytes. +fn strings<'a>(values: impl Iterator) -> (Vec, Vec) { + let mut offsets = vec![0]; + let mut data = Vec::new(); + for value in values { + data.extend_from_slice(value.as_bytes()); + offsets.push(data.len() as i32); + } + (offsets, data) +} + +fn str_column<'a>(name: &'a str, (offsets, data): &'a (Vec, Vec)) -> Column<'a> { + Column { + name: Some(name), + data: ColumnData::Str { offsets, data }, + validity: None, + } +} + fn benchmark_write(criterion: &mut Criterion) { let rows = load_rows(); - let owned: Vec = - ::to_columns(&rows).expect("transpose must succeed"); - let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); + let region = strings(rows.iter().map(|r| r.region.as_str())); + let country = strings(rows.iter().map(|r| r.country.as_str())); + let item_type = strings(rows.iter().map(|r| r.item_type.as_str())); + let order_date: Vec = rows.iter().map(|r| r.order_date.days_since_epoch).collect(); + let order_id: Vec = rows.iter().map(|r| r.order_id).collect(); + let units_sold: Vec = rows.iter().map(|r| r.units_sold).collect(); + let total_revenue: Vec = rows.iter().map(|r| r.total_revenue).collect(); + let column = |name, data| Column { + name: Some(name), + data, + validity: None, + }; + let borrowed = [ + str_column("Region", ®ion), + str_column("Country", &country), + str_column("Item Type", &item_type), + column("Order Date", ColumnData::Date(&order_date)), + column("Order ID", ColumnData::I64(&order_id)), + column("Units Sold", ColumnData::I64(&units_sold)), + column("Total Revenue", ColumnData::F64(&total_revenue)), + ]; + let options = WriteOptions::new(); let mut group = criterion.benchmark_group("write_xlsx_65k"); group.sample_size(20); group.bench_function("columns", |b| { let path = output_path("columns.xlsx"); - let target = path.to_str().expect("temp path must be UTF-8"); b.iter(|| { - write_columns(target, XL_FORMAT_XLSX, std::hint::black_box(&borrowed), None) + write_columns(&path, std::hint::black_box(&borrowed), &options) .expect("write_columns must succeed"); }); std::fs::remove_file(&path).ok(); @@ -86,9 +127,8 @@ fn benchmark_write(criterion: &mut Criterion) { group.bench_function("sheet", |b| { let path = output_path("sheet.xlsx"); - let target = path.to_str().expect("temp path must be UTF-8"); b.iter(|| { - write_sheet(target, XL_FORMAT_XLSX, std::hint::black_box(&rows), None) + write_sheet(&path, std::hint::black_box(&rows), &options) .expect("write_sheet must succeed"); }); std::fs::remove_file(&path).ok(); diff --git a/rust/excelreader/src/aggregate.rs b/rust/excelreader/src/aggregate.rs index cfa85db8..08a3a4c7 100644 --- a/rust/excelreader/src/aggregate.rs +++ b/rust/excelreader/src/aggregate.rs @@ -7,13 +7,13 @@ //! can unwind into the library, and resumes that panic on the calling thread once the run has //! finished freeing every state. -use crate::{ - rows::RowRef, workbook, Error, XlCsvAggregation, XlCsvParallelOptions, XlRow, XL_OK, -}; +use crate::ffi::{self, XlCsvAggregation, XlRow, XL_OK}; +use crate::workbook::{self, len_i32, path_str}; +use crate::{Error, ErrorKind, OpenOptions, RowRef}; use std::any::Any; use std::ffi::c_void; use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Mutex; /// The status a shim returns to signal "a callback panicked". The library treats it like any other @@ -47,7 +47,8 @@ struct State { /// library later re-reads the partition, the failure is discarded with it and the run continues; /// otherwise the run ends with that code. A nonzero code from `combine`, or a panic from the seed /// closure, always ends the run. However it ends, the code is returned to the caller verbatim as -/// [`Error::code`]; use positive codes, since every code the library itself returns is `<= 0`. +/// [`ErrorKind::Callback`]; use positive codes, since every code the library itself returns is +/// `<= 0`. /// Sibling workers are never stopped, so both `accumulate` and `combine` must tolerate being called /// again after returning an error. pub trait CsvAccumulator: Send { @@ -58,30 +59,41 @@ pub trait CsvAccumulator: Send { fn combine(&mut self, other: &mut Self) -> Result<(), i32>; } -/// Mirrors `xl_csv_parallel_options`. Every zero field means "library default"; in particular -/// `degree_of_parallelism` 0 is the processor count and 1 forces a sequential run, and `header_row` -/// is 1-based with 0 meaning "no header". -#[derive(Clone, Copy, Debug, Default)] -pub struct CsvParallelOptions { - pub degree_of_parallelism: i32, - pub header_row: i32, - pub delimiter: i32, - pub quote: i32, - pub detect_bom: i32, - pub max_cell_bytes: i32, +/// What [`aggregate_csv`] reads: a CSV file, or CSV bytes already in memory. A `&str` converts to a +/// path, never to CSV text - pass `text.as_bytes()` for that. +#[derive(Clone, Copy, Debug)] +pub enum CsvInput<'a> { + Path(&'a Path), + Bytes(&'a [u8]), } -impl CsvParallelOptions { - fn to_raw(self) -> XlCsvParallelOptions { - XlCsvParallelOptions { - struct_size: std::mem::size_of::() as i32, - degree_of_parallelism: self.degree_of_parallelism, - header_row: self.header_row, - delimiter: self.delimiter, - quote: self.quote, - detect_bom: self.detect_bom, - max_cell_bytes: self.max_cell_bytes, - } +impl<'a> From<&'a Path> for CsvInput<'a> { + fn from(path: &'a Path) -> Self { + CsvInput::Path(path) + } +} + +impl<'a> From<&'a PathBuf> for CsvInput<'a> { + fn from(path: &'a PathBuf) -> Self { + CsvInput::Path(path) + } +} + +impl<'a> From<&'a str> for CsvInput<'a> { + fn from(path: &'a str) -> Self { + CsvInput::Path(Path::new(path)) + } +} + +impl<'a> From<&'a [u8]> for CsvInput<'a> { + fn from(bytes: &'a [u8]) -> Self { + CsvInput::Bytes(bytes) + } +} + +impl<'a, const N: usize> From<&'a [u8; N]> for CsvInput<'a> { + fn from(bytes: &'a [u8; N]) -> Self { + CsvInput::Bytes(bytes) } } @@ -205,18 +217,12 @@ fn shared(seed: &F) -> Shared<'_, A, F> { Shared { seed, panic: Mutex::new(None), _marker: std::marker::PhantomData } } -/// The ABI takes every length as `int32`, so an oversized input is rejected rather than truncated -/// into a silently partial - but successful - aggregation. -fn length(len: usize, what: &str) -> Result { - i32::try_from(len).map_err(|_| { - Error::from_status( - crate::XL_INVALID_ARGUMENT, - format!("{what} is {len} bytes, past the ABI's int32 length limit"), - ) - }) -} - -/// Folds the CSV file at `path` into one `A` across several threads. +/// Folds a CSV file or buffer into one `A` across several threads. +/// +/// The CSV dialect (`csv_delimiter`, `csv_quote`, `csv_detect_bom`) and `max_cell_bytes` come from +/// `options`; its other settings do not apply. `header_row` is 1-based, `0` meaning no header. +/// `degree_of_parallelism` is `0` for every processor, `1` sequential, `n` up to `n` threads. +/// A buffer is not copied; the borrow keeps it valid for the call. /// /// `seed` is called at least once per partition, on worker threads, and may be called again for a /// partition that has to be re-read - a seeded accumulator can therefore be dropped without ever @@ -226,85 +232,54 @@ fn length(len: usize, what: &str) -> Result { /// The call blocks until the run finishes, so `seed` may borrow locals freely: there is no `'static` /// bound and none is needed. A panic inside any callback aborts the run and is resumed here, on the /// calling thread, once the library has freed every state. -pub fn aggregate_csv_file( - path: &Path, - seed: F, - options: &CsvParallelOptions, -) -> Result -where - A: CsvAccumulator, - F: Fn() -> A + Sync, -{ - workbook::check_abi_version()?; - let bytes = path - .to_str() - .ok_or_else(|| { - Error::from_status( - crate::XL_INVALID_ARGUMENT, - format!("path {} is not valid UTF-8", path.display()), - ) - })? - .as_bytes(); - let path_len = length(bytes.len(), "path")?; - let shared = shared::(&seed); - let agg = raw_aggregation(&shared); - let raw_options = options.to_raw(); - let mut state: *mut c_void = std::ptr::null_mut(); - - let status = unsafe { - crate::xl_csv_aggregate_file( - bytes.as_ptr(), - path_len, - &agg, - &raw_options, - &mut state, - ) - }; - finish(shared, status, state) -} - -/// Folds an in-memory CSV buffer into one `A`. The buffer is not copied and must stay valid for the -/// duration of the call, which this signature guarantees. Otherwise identical to -/// [`aggregate_csv_file`]. -pub fn aggregate_csv_memory( - data: &[u8], +pub fn aggregate_csv<'a, A, F>( + input: impl Into>, + options: &OpenOptions, + header_row: i32, + degree_of_parallelism: i32, seed: F, - options: &CsvParallelOptions, ) -> Result where A: CsvAccumulator, F: Fn() -> A + Sync, { workbook::check_abi_version()?; - aggregate_csv_memory_unchecked(data, seed, options) + aggregate_unchecked(input.into(), options, header_row, degree_of_parallelism, seed) } -/// [`aggregate_csv_memory`] without the ABI check, for the warm-up run in +/// [`aggregate_csv`] without the ABI check, for the warm-up run in /// [`workbook::check_abi_version`] itself - calling the public entry point from there would /// re-enter its `OnceLock` and deadlock. -pub(crate) fn aggregate_csv_memory_unchecked( - data: &[u8], +pub(crate) fn aggregate_unchecked( + input: CsvInput<'_>, + options: &OpenOptions, + header_row: i32, + degree_of_parallelism: i32, seed: F, - options: &CsvParallelOptions, ) -> Result where A: CsvAccumulator, F: Fn() -> A + Sync, { - let data_len = length(data.len(), "buffer")?; + let raw_options = options.to_csv_parallel_raw(header_row, degree_of_parallelism); let shared = shared::(&seed); let agg = raw_aggregation(&shared); - let raw_options = options.to_raw(); let mut state: *mut c_void = std::ptr::null_mut(); - let status = unsafe { - crate::xl_csv_aggregate_memory( - data.as_ptr(), - data_len, - &agg, - &raw_options, - &mut state, - ) + let status = match input { + CsvInput::Path(path) => { + let path = path_str(path)?; + let path_len = len_i32(path.len(), "path")?; + unsafe { + ffi::xl_csv_aggregate_file(path.as_ptr(), path_len, &agg, &raw_options, &mut state) + } + } + CsvInput::Bytes(data) => { + let data_len = len_i32(data.len(), "buffer")?; + unsafe { + ffi::xl_csv_aggregate_memory(data.as_ptr(), data_len, &agg, &raw_options, &mut state) + } + } }; finish(shared, status, state) } @@ -344,9 +319,9 @@ where return Err(status_error(status)); } winner.map(|boxed| boxed.value).ok_or_else(|| { - Error::from_status( - crate::XL_ERROR, - "native reported success but wrote no aggregation state".to_string(), + Error::new( + ErrorKind::Other, + "native reported success but wrote no aggregation state", ) }) } @@ -356,7 +331,10 @@ where /// message for. fn status_error(status: i32) -> Error { if status > 0 { - Error::from_status(status, format!("an aggregation callback returned status {status}")) + Error::new( + ErrorKind::Callback(status), + format!("an aggregation callback returned status {status}"), + ) } else { workbook::last_error(status) } diff --git a/rust/excelreader/src/arrow.rs b/rust/excelreader/src/arrow.rs index 72130276..02a0003f 100644 --- a/rust/excelreader/src/arrow.rs +++ b/rust/excelreader/src/arrow.rs @@ -13,57 +13,88 @@ use arrow::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use arrow::record_batch::RecordBatchReader; -use crate::workbook::{build_specs, check, ExcelMapper, Sheet, Workbook}; -use crate::{Error, XL_ERROR}; - -/// Schema-driven parse of one sheet into an Arrow [`RecordBatch`], using the same -/// `#[derive(ExcelMapper)]` mapping as [`crate::workbook::parse_sheet`]. -/// -/// `header_row` has the same meaning as in `parse_sheet` (0 = no header). -pub fn parse_arrow(sheet: Sheet<'_>, header_row: i32) -> Result { - parse_arrow_parallel::(sheet, header_row, 1) -} +use crate::ffi; +use crate::workbook::{build_specs, check}; +use crate::{Error, ErrorKind, ExcelMapper, Sheet, Workbook}; + +impl<'w> Sheet<'w> { + /// [`parse`](Self::parse) into one Arrow [`RecordBatch`], with the same `#[derive(ExcelMapper)]` + /// mapping, `header_row` and `degree_of_parallelism`. + pub fn parse_arrow( + &self, + header_row: i32, + degree_of_parallelism: i32, + ) -> Result { + let arena = build_specs::(); + + let mut array = FFI_ArrowArray::empty(); + let mut schema = FFI_ArrowSchema::empty(); + + check(unsafe { + ffi::xl_parse_arrow( + self.handle(), + self.index(), + arena.specs.as_ptr(), + arena.specs.len() as i32, + header_row, + degree_of_parallelism, + &mut array as *mut FFI_ArrowArray as *mut c_void, + &mut schema as *mut FFI_ArrowSchema as *mut c_void, + ) + })?; + + let data = unsafe { from_ffi(array, &schema) }.map_err(|e| { + Error::new( + ErrorKind::Other, + format!("importing the native Arrow array failed: {e}"), + ) + })?; + + Ok(RecordBatch::from(StructArray::from(data))) + } -/// [`parse_arrow`] with a degree of parallelism: `0` uses every processor, `1` is sequential, `n` -/// up to `n` threads, negative is an error. Only CSV is split; the batch equals `parse_arrow`'s. -pub fn parse_arrow_parallel( - sheet: Sheet<'_>, - header_row: i32, - degree_of_parallelism: i32, -) -> Result { - let arena = build_specs::(); - - let mut array = FFI_ArrowArray::empty(); - let mut schema = FFI_ArrowSchema::empty(); - - check(unsafe { - crate::xl_parse_arrow( - sheet.handle(), - sheet.index(), - arena.specs.as_ptr(), - arena.specs.len() as i32, - header_row, - degree_of_parallelism, - &mut array as *mut FFI_ArrowArray as *mut c_void, - &mut schema as *mut FFI_ArrowSchema as *mut c_void, - ) - })?; - - let data = unsafe { from_ffi(array, &schema) }.map_err(|e| { - Error::from_status( - XL_ERROR, - format!("importing the native Arrow array failed: {e}"), - ) - })?; - - Ok(RecordBatch::from(StructArray::from(data))) + /// [`parse_arrow`](Self::parse_arrow) delivered a batch at a time. `batch_size` is rows per + /// batch, 0 unbounded, negative an error. Any number of streams may be open on one workbook. + pub fn parse_arrow_stream( + &self, + header_row: i32, + batch_size: i64, + ) -> Result, Error> { + let arena = build_specs::(); + + let mut stream = FFI_ArrowArrayStream::empty(); + + check(unsafe { + ffi::xl_parse_arrow_stream( + self.handle(), + self.index(), + arena.specs.as_ptr(), + arena.specs.len() as i32, + header_row, + batch_size, + &mut stream as *mut FFI_ArrowArrayStream as *mut c_void, + ) + })?; + + let inner = ArrowArrayStreamReader::try_new(stream).map_err(|e| { + Error::new( + ErrorKind::Other, + format!("importing the native Arrow stream failed: {e}"), + ) + })?; + + Ok(ArrowChunks { + inner, + _workbook: PhantomData, + }) + } } -/// Batched counterpart to [`parse_arrow`], one [`RecordBatch`] at a time. +/// Batches of [`Sheet::parse_arrow_stream`], one [`RecordBatch`] at a time. /// -/// Wraps arrow-rs's `'static` reader only to carry the `'a` borrow, which keeps the -/// workbook alive for at least as long as the stream. Unlike [`crate::workbook::TypedChunks`] it does -/// not fuse after an error, so break on the first `Err` rather than spinning on the latched one. +/// Wraps arrow-rs's `'static` reader only to carry the `'a` borrow, which keeps the workbook alive +/// for at least as long as the stream. Unlike [`TypedChunks`](crate::TypedChunks) it does not fuse +/// after an error, so break on the first `Err` rather than spinning on the latched one. pub struct ArrowChunks<'a> { inner: ArrowArrayStreamReader, _workbook: PhantomData<&'a Workbook>, @@ -82,39 +113,3 @@ impl RecordBatchReader for ArrowChunks<'_> { self.inner.schema() } } - -/// `parse_arrow` delivered a batch at a time. `batch_size` is rows per batch, 0 unbounded, -/// negative an error. Any number of streams may be open on one workbook. -pub fn parse_arrow_stream( - sheet: Sheet<'_>, - header_row: i32, - batch_size: i64, -) -> Result, Error> { - let arena = build_specs::(); - - let mut stream = FFI_ArrowArrayStream::empty(); - - check(unsafe { - crate::xl_parse_arrow_stream( - sheet.handle(), - sheet.index(), - arena.specs.as_ptr(), - arena.specs.len() as i32, - header_row, - batch_size, - &mut stream as *mut FFI_ArrowArrayStream as *mut c_void, - ) - })?; - - let inner = ArrowArrayStreamReader::try_new(stream).map_err(|e| { - Error::from_status( - XL_ERROR, - format!("importing the native Arrow stream failed: {e}"), - ) - })?; - - Ok(ArrowChunks { - inner, - _workbook: PhantomData, - }) -} diff --git a/rust/excelreader/src/error.rs b/rust/excelreader/src/error.rs index e3d0ac2e..aa428eeb 100644 --- a/rust/excelreader/src/error.rs +++ b/rust/excelreader/src/error.rs @@ -1,67 +1,90 @@ use std::fmt; -/// A native ExcelReader error. `code` is one of the `XL_*` status constants; `message` is the -/// detail from `xl_last_error_ptr` on the calling thread at the time of failure. -/// -/// [`PasswordRequired`](Error::PasswordRequired) and [`PasswordIncorrect`](Error::PasswordIncorrect) -/// get their own variants - rather than staying folded into a generic `code`/`message` pair - so -/// callers can `match`/`matches!` on "needs a password" without hardcoding the `-6`/`-7` status -/// constants themselves. Every other status stays in [`Native`](Error::Native), which is where the -/// distinction is drawn: not "every code gets a variant" but "every code a caller needs to branch on -/// by name gets one". -#[derive(Debug, Clone)] -pub enum Error { - /// Any native status not covered by a more specific variant below. - Native { code: i32, message: String }, - /// The workbook is encrypted and no password was supplied (`XL_STATUS_PASSWORD_REQUIRED`). - PasswordRequired { message: String }, - /// The supplied password did not match the workbook's verifier (`XL_STATUS_PASSWORD_INCORRECT`). - PasswordIncorrect { message: String }, +use crate::ffi; + +/// What went wrong, for callers that branch on the failure. The human-readable detail is +/// [`Error::message`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ErrorKind { + /// An argument was rejected before any work was done: a bad option value, an out-of-range + /// index, mismatched column lengths, a path that is not UTF-8. + InvalidArgument, + /// The workbook is encrypted and no password was supplied. + PasswordRequired, + /// The supplied password did not match the workbook's verifier. + PasswordIncorrect, + /// A [`CsvAccumulator`](crate::CsvAccumulator) callback returned this code, passed through + /// verbatim. + Callback(i32), + /// Anything else: an I/O failure, a malformed or unsupported file, a resource limit. + Other, +} + +/// An ExcelReader error: an [`ErrorKind`] plus the detail message the native library (or this +/// crate) reported at the time of failure. +#[derive(Clone, Debug)] +pub struct Error { + kind: ErrorKind, + message: String, } impl Error { - /// Builds an [`Error`] from a raw `XL_*` status code and a detail message, routing - /// `XL_STATUS_PASSWORD_REQUIRED`/`XL_STATUS_PASSWORD_INCORRECT` to their own variants and - /// everything else to [`Native`](Error::Native). The one place that mapping happens, so every - /// call site that turns a status code into an `Error` (here, in `arrow`/`writer`, and in - /// `excelreader_derive`'s generated `append_tokens`) goes through it instead of re-deriving the - /// mapping. `pub`, not `pub(crate)`: the derive macro expands into the crate that invokes it, - /// not this one, so its generated code needs a public constructor to build a well-formed `Error` - /// without reaching into a private field or duplicating this match. - #[must_use] - pub fn from_status(code: i32, message: String) -> Self { - match code { - crate::XL_STATUS_PASSWORD_REQUIRED => Error::PasswordRequired { message }, - crate::XL_STATUS_PASSWORD_INCORRECT => Error::PasswordIncorrect { message }, - _ => Error::Native { code, message }, + pub(crate) fn new(kind: ErrorKind, message: impl Into) -> Self { + Error { + kind, + message: message.into(), } } - /// The `XL_*` status constant this error carries. + /// The one place a native status code becomes an [`ErrorKind`]. Positive codes only ever come + /// from an aggregation callback; every library status is `<= 0`. + pub(crate) fn from_status(code: i32, message: String) -> Self { + let kind = match code { + ffi::XL_INVALID_ARGUMENT => ErrorKind::InvalidArgument, + ffi::XL_STATUS_PASSWORD_REQUIRED => ErrorKind::PasswordRequired, + ffi::XL_STATUS_PASSWORD_INCORRECT => ErrorKind::PasswordIncorrect, + code if code > 0 => ErrorKind::Callback(code), + _ => ErrorKind::Other, + }; + Error { kind, message } + } + + pub(crate) fn io(error: &std::io::Error) -> Self { + Error::new(ErrorKind::Other, error.to_string()) + } + #[must_use] - pub fn code(&self) -> i32 { - match self { - Error::Native { code, .. } => *code, - Error::PasswordRequired { .. } => crate::XL_STATUS_PASSWORD_REQUIRED, - Error::PasswordIncorrect { .. } => crate::XL_STATUS_PASSWORD_INCORRECT, - } + pub fn kind(&self) -> ErrorKind { + self.kind } - /// The detail from `xl_last_error_ptr` at the time of failure. + /// The detail reported at the time of failure. #[must_use] pub fn message(&self) -> &str { - match self { - Error::Native { message, .. } - | Error::PasswordRequired { message } - | Error::PasswordIncorrect { message } => message, - } + &self.message } } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ExcelReader error {}: {}", self.code(), self.message()) + write!(f, "ExcelReader error ({:?}): {}", self.kind, self.message) } } impl std::error::Error for Error {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn statuses_map_onto_kinds() { + let kind = |code| Error::from_status(code, String::new()).kind(); + assert_eq!(kind(ffi::XL_INVALID_ARGUMENT), ErrorKind::InvalidArgument); + assert_eq!(kind(ffi::XL_STATUS_PASSWORD_REQUIRED), ErrorKind::PasswordRequired); + assert_eq!(kind(ffi::XL_STATUS_PASSWORD_INCORRECT), ErrorKind::PasswordIncorrect); + assert_eq!(kind(42), ErrorKind::Callback(42)); + assert_eq!(kind(-5), ErrorKind::Other); + } +} diff --git a/rust/excelreader/src/ffi.rs b/rust/excelreader/src/ffi.rs new file mode 100644 index 00000000..8cdc9fa7 --- /dev/null +++ b/rust/excelreader/src/ffi.rs @@ -0,0 +1,495 @@ +//! Raw FFI bindings to ExcelReader's C ABI (`excelreader.h`). Crate-private: every public type is a +//! safe wrapper over these. + +use std::os::raw::{c_int, c_void}; + +pub const XL_OK: i32 = 0; +pub const XL_EOF: i32 = -1; +pub const XL_BUFFER_TOO_SMALL: i32 = -2; +pub const XL_INVALID_ARGUMENT: i32 = -4; +/// The workbook is encrypted and no password was supplied. +pub const XL_STATUS_PASSWORD_REQUIRED: i32 = -6; +/// The supplied password did not match the workbook's verifier. +pub const XL_STATUS_PASSWORD_INCORRECT: i32 = -7; + +pub const XL_CELL_EMPTY: i32 = 0; +pub const XL_CELL_STRING: i32 = 1; +pub const XL_CELL_NUMBER: i32 = 2; +pub const XL_CELL_DATE: i32 = 3; +pub const XL_CELL_BOOL: i32 = 4; +pub const XL_CELL_FORMULA: i32 = 5; +pub const XL_CELL_ERROR: i32 = 6; + +/// ABI revision this crate is compiled against. Every constructor refuses to proceed when the loaded +/// library's `xl_abi_version()` disagrees - see `workbook::check_abi_version`. +pub const XL_ABI_VERSION: i32 = 6; + +pub const XL_T_STRING: i32 = 0; +pub const XL_T_I64: i32 = 1; +pub const XL_T_F64: i32 = 2; +pub const XL_T_BOOL: i32 = 3; +/// Days since 1970-01-01, stored as `i32`. +pub const XL_T_DATE: i32 = 4; +/// Microseconds since midnight, stored as `i64`. +pub const XL_T_TIME: i32 = 5; +/// Microseconds since 1970-01-01T00:00:00Z, stored as `i64`. +pub const XL_T_TIMESTAMP: i32 = 6; + +/// `xl_infer_schema` flag: also type cells that hold text (integers, decimals, booleans, ISO dates). +pub const XL_INFER_PARSE_TEXT: i32 = 1; + +pub const XL_SHEET_VISIBLE: i32 = 0; +pub const XL_SHEET_HIDDEN: i32 = 1; +pub const XL_SHEET_VERY_HIDDEN: i32 = 2; + +pub const XL_FORMAT_AUTO: i32 = 0; +pub const XL_FORMAT_XLS: i32 = 1; +pub const XL_FORMAT_XLSX: i32 = 2; +pub const XL_FORMAT_XLSB: i32 = 3; +pub const XL_FORMAT_CSV: i32 = 4; + +/// Every boolean-shaped `xl_open_options` field uses one of these three states, never a plain 0/1 - +/// several of them default to true, so a bare 0 would be ambiguous between "off" and "use the +/// library default". +pub const XL_OPT_DEFAULT: i32 = 0; +pub const XL_OPT_FALSE: i32 = 1; +pub const XL_OPT_TRUE: i32 = 2; + +/// Opaque handle - never dereferenced by Rust, only passed back to `xl_*` functions. +#[repr(C)] +pub struct XlWorkbook { + _private: [u8; 0], +} + +/// Opaque row cursor (`xl_rows`) - never dereferenced by Rust, only passed back to `xl_rows_*`. +#[repr(C)] +pub struct XlRows { + _private: [u8; 0], +} + +#[repr(C)] +pub struct XlTypedReader { + _private: [u8; 0], +} + +/// Mirrors `xl_open_options`. `struct_size` must be set to `size_of::()`; build one +/// through `OpenOptions` rather than by hand. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlOpenOptions { + pub struct_size: i32, + pub csv_sniff_dialect: i32, + pub csv_delimiter: i32, + pub csv_quote: i32, + pub csv_detect_bom: i32, + pub csv_max_cell_bytes: i32, + pub csv_intern_strings: i32, + pub max_total_decompressed_bytes: i64, + pub max_cell_bytes: i32, + pub max_shared_string_bytes: i64, + pub max_zip_entries: i32, + pub prefetch_decompression: i32, + pub intern_strings: i32, + + /// Password for an encrypted OOXML workbook, as UTF-8 bytes. NULL (with `password_len` 0) means + /// "not encrypted, or fail with `XL_STATUS_PASSWORD_REQUIRED`". Not NUL-terminated - + /// `password_len` is authoritative, since a password may contain any byte. The pointer need only + /// remain valid for the duration of the call - see `options::OpenOptionsRaw`. + pub password: *const u8, + pub password_len: i32, + pub source_cache_bytes: i64, + pub max_buffered_bytes: i64, + pub source_block_size: i32, + pub verify_zip_checksums: i32, +} + +#[repr(C)] +pub struct XlSource { + pub struct_size: i32, + pub user_data: *mut c_void, + pub length: i64, + pub read_at: Option i64>, + pub release: Option, +} + +#[repr(C)] +pub struct XlStream { + pub struct_size: i32, + pub user_data: *mut c_void, + pub read: Option i64>, + pub release: Option, +} + +/// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with +/// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts +/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. +/// Build one through `WriteOptions` rather than by hand. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlWriteOptions { + pub struct_size: i32, + pub sheet_name_len: i32, + pub sheet_name: *const u8, + pub csv_delimiter: i32, + pub csv_quote: i32, + pub date1904: i32, + pub use_shared_strings: i32, +} + +#[repr(C)] +pub struct XlColumnSpec { + pub names: *const *const u8, + pub name_lens: *const i32, + pub name_count: i32, + pub index: i32, + pub r#type: i32, + pub nullable: i32, +} + +/// One column of a native table. Reachable from `__private` because the derive's generated +/// bindings read it through the `column_*` accessors. +#[repr(C)] +pub struct XlColumn { + pub r#type: i32, + pub length: i64, + pub values: *const c_void, + pub validity: *const u8, + pub data: *const u8, + pub data_len: i64, +} + +#[repr(C)] +pub struct XlTable { + pub column_count: i32, + pub row_count: i64, + pub columns: *mut XlColumn, +} + +/// Mirrors `xl_inferred_schema`. Owned by the native library until `xl_free_schema`. +#[repr(C)] +pub struct XlInferredSchema { + pub columns: *mut XlColumnSpec, + pub column_count: i32, +} + +/// Mirrors `xl_buffer`: an owned block of native memory returned by `xl_write_typed_to_memory` or +/// `xl_write_handle_bytes`. Released with `xl_free_buffer` - see `workbook::buffer_to_vec`, the one +/// place this crate touches the raw struct directly. +#[repr(C)] +pub struct XlBuffer { + pub data: *mut u8, + pub len: i64, +} + +/// Opaque streaming writer handle - never dereferenced by Rust, only passed back to `xl_*` +/// functions. +#[repr(C)] +pub struct XlWriterHandle { + _private: [u8; 0], +} + +/// Mirrors `xl_row_cell`. Named `cell_type` because `type` is a Rust keyword. +#[repr(C)] +#[derive(Debug)] +pub struct XlRowCell { + pub column: i32, + pub cell_type: i32, + pub value_len: i32, + pub value: *const u8, +} + +/// Mirrors `xl_row`. +#[repr(C)] +pub struct XlRow { + pub cell_count: i32, + pub cells: *mut XlRowCell, +} + +/// Mirrors `xl_csv_aggregation`. Every callback runs on a library worker thread except +/// `free_state`, which runs on the calling thread after the run has finished; none may let a +/// panic escape into the library - build one through `aggregate` rather than by hand. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XlCsvAggregation { + pub struct_size: i32, + pub seed: Option i32>, + pub accumulate: Option i32>, + pub combine: Option i32>, + pub free_state: Option, + pub user_data: *mut c_void, +} + +/// Mirrors `xl_csv_parallel_options`. `struct_size` must be set to +/// `size_of::()`; built by `OpenOptions::to_csv_parallel_raw`. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct XlCsvParallelOptions { + pub struct_size: i32, + pub degree_of_parallelism: i32, + pub header_row: i32, + pub delimiter: i32, + pub quote: i32, + pub detect_bom: i32, + pub max_cell_bytes: i32, +} + +extern "C" { + pub fn xl_abi_version() -> c_int; + + pub fn xl_open_file( + path: *const u8, + path_len: i32, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_memory( + data: *const u8, + data_len: i32, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_source( + source: *const XlSource, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_stream( + stream: *const XlStream, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_set_source_error(message: *const u8, len: i32); + + pub fn xl_close(handle: *mut XlWorkbook) -> c_int; + + pub fn xl_sheet_count(handle: *mut XlWorkbook, out_count: *mut i32) -> c_int; + + pub fn xl_sheet_name_at( + handle: *mut XlWorkbook, + index: i32, + buffer: *mut u8, + capacity: i32, + out_len: *mut i32, + ) -> c_int; + + pub fn xl_sheet_visibility_at( + handle: *mut XlWorkbook, + index: i32, + out_visibility: *mut i32, + ) -> c_int; + + /// A name that matches no sheet is `XL_OK` with `-1`. + pub fn xl_sheet_index( + handle: *mut XlWorkbook, + name: *const u8, + name_len: i32, + out_index: *mut i32, + ) -> c_int; + + pub fn xl_is_date1904(handle: *mut XlWorkbook, out_flag: *mut i32) -> c_int; + + pub fn xl_parse_typed( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + degree_of_parallelism: i32, + out_table: *mut XlTable, + ) -> c_int; + + /// Same schema-driven parse as `xl_parse_typed`, exported as one top-level Arrow struct + /// array/schema. `out_array`/`out_schema` are `struct ArrowArray*`/`struct ArrowSchema*` from + /// the Arrow C Data Interface, typed here as `c_void` because arrow-rs's own `#[repr(C)]` + /// `FFI_ArrowArray`/`FFI_ArrowSchema` are ABI-identical to them - redeclaring the spec structs + /// would be a second source of truth for a fixed, versioned ABI. + /// + /// On `XL_OK` the caller owns both and releases each through its OWN `release` callback, never + /// through `xl_free_table`. On any other status both outputs are left untouched. + #[cfg(feature = "arrow")] + pub fn xl_parse_arrow( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + degree_of_parallelism: i32, + out_array: *mut c_void, + out_schema: *mut c_void, + ) -> c_int; + + /// `out_stream` is a `struct ArrowArrayStream*`, typed as `c_void` for the same reason + /// `xl_parse_arrow`'s outputs are. Its `release` is what closes the underlying read. + #[cfg(feature = "arrow")] + pub fn xl_parse_arrow_stream( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + max_rows: i64, + out_stream: *mut c_void, + ) -> c_int; + + pub fn xl_free_table(table: *mut XlTable); + + /// `max_rows` is rows per batch, 0 meaning one unbounded batch. + pub fn xl_typed_reader_open( + handle: *mut XlWorkbook, + sheet: i32, + specs: *const XlColumnSpec, + spec_count: i32, + header_row: i32, + max_rows: i64, + out_reader: *mut *mut XlTypedReader, + ) -> c_int; + + /// `XL_OK` with a batch, or `XL_EOF` at end of sheet. A failure latches: every later call + /// repeats it. + pub fn xl_typed_reader_next(reader: *mut XlTypedReader, out_table: *mut XlTable) -> c_int; + + pub fn xl_typed_reader_close(reader: *mut XlTypedReader); + + pub fn xl_infer_schema( + handle: *mut XlWorkbook, + sheet: i32, + header_row: i32, + sample_size: i32, + flags: i32, + out_schema: *mut XlInferredSchema, + ) -> c_int; + + pub fn xl_free_schema(schema: *mut XlInferredSchema); + pub fn xl_write_typed( + path: *const u8, + path_len: i32, + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + ) -> c_int; + + /// Same as `xl_write_typed`, except the result is returned as `out_buffer` instead of being + /// written to a path. Only read `*out_buffer` when the call returns `XL_OK`; on failure it is + /// zeroed, and `xl_free_buffer` on a zeroed buffer is a no-op. + pub fn xl_write_typed_to_memory( + format: i32, + specs: *const XlColumnSpec, + table: *const XlTable, + options: *const XlWriteOptions, + out_buffer: *mut XlBuffer, + ) -> c_int; + + /// Releases a buffer returned by `xl_write_typed_to_memory` or `xl_write_handle_bytes` and + /// resets it to zero. Safe on a zeroed value. + pub fn xl_free_buffer(buffer: *mut XlBuffer); + + /// Reads the plaintext XLSX/XLSB package at `package_path` and writes its agile-encrypted + /// (ECMA-376 4.4) counterpart to `destination_path`, overwriting an existing file. The result + /// opens with the same password via `xl_open_file`'s `xl_open_options::password`. + pub fn xl_encrypt_package( + package_path: *const u8, + package_path_len: i32, + destination_path: *const u8, + destination_path_len: i32, + password: *const u8, + password_len: i32, + ) -> c_int; + + pub fn xl_open_write_handle( + path: *const u8, + path_len: i32, + format: i32, + options: *const XlWriteOptions, + out_handle: *mut *mut XlWriterHandle, + ) -> c_int; + + pub fn xl_open_write_handle_to_memory( + format: i32, + options: *const XlWriteOptions, + out_handle: *mut *mut XlWriterHandle, + ) -> c_int; + + pub fn xl_start_sheet(handle: *mut XlWriterHandle, name: *const u8, name_len: i32) -> c_int; + pub fn xl_start_row(handle: *mut XlWriterHandle) -> c_int; + + pub fn xl_write_string(handle: *mut XlWriterHandle, value: *const u8, value_len: i32) -> c_int; + pub fn xl_write_int64(handle: *mut XlWriterHandle, value: i64) -> c_int; + pub fn xl_write_float64(handle: *mut XlWriterHandle, value: f64) -> c_int; + pub fn xl_write_bool(handle: *mut XlWriterHandle, value: i32) -> c_int; + pub fn xl_write_date(handle: *mut XlWriterHandle, days_since_epoch: i32) -> c_int; + pub fn xl_write_time(handle: *mut XlWriterHandle, micros_since_midnight: i64) -> c_int; + pub fn xl_write_timestamp(handle: *mut XlWriterHandle, micros_since_epoch: i64) -> c_int; + pub fn xl_write_null(handle: *mut XlWriterHandle, r#type: i32) -> c_int; + + pub fn xl_end_row(handle: *mut XlWriterHandle) -> c_int; + pub fn xl_end_sheet(handle: *mut XlWriterHandle) -> c_int; + pub fn xl_close_write_handle(handle: *mut XlWriterHandle) -> c_int; + + /// Reads back everything written so far to a handle opened by `xl_open_write_handle_to_memory`. + /// `XL_INVALID_ARGUMENT` for one from `xl_open_write_handle`. Only read `*out_buffer` when the + /// call returns `XL_OK`; on failure it is zeroed, and `xl_free_buffer` on a zeroed buffer is a + /// no-op. + pub fn xl_write_handle_bytes(handle: *mut XlWriterHandle, out_buffer: *mut XlBuffer) -> c_int; + + pub fn xl_rows_open(handle: *mut XlWorkbook, sheet: i32, out_rows: *mut *mut XlRows) -> c_int; + pub fn xl_rows_close(rows: *mut XlRows) -> c_int; + pub fn xl_rows_next( + rows: *mut XlRows, + buffer: *mut u8, + capacity: i32, + out_written: *mut i32, + ) -> c_int; + pub fn xl_rows_read_all_blob( + rows: *mut XlRows, + buffer: *mut u8, + capacity: i32, + out_written: *mut i32, + ) -> c_int; + + /// Folds a CSV file into one caller-owned accumulator across several threads. Blocks until the + /// run finishes; the surviving state is written to `out_state` only on `XL_OK` and becomes the + /// caller's to free. See the contract above `xl_csv_aggregation` in `excelreader.h`, and + /// `aggregate::aggregate_csv` for the safe wrapper. + pub fn xl_csv_aggregate_file( + path: *const u8, + path_len: i32, + agg: *const XlCsvAggregation, + options: *const XlCsvParallelOptions, + out_state: *mut *mut c_void, + ) -> c_int; + + /// Same as `xl_csv_aggregate_file` over an in-memory buffer, which must stay valid and + /// unmodified until the call returns - the library does not copy it. + pub fn xl_csv_aggregate_memory( + data: *const u8, + data_len: i32, + agg: *const XlCsvAggregation, + options: *const XlCsvParallelOptions, + out_state: *mut *mut c_void, + ) -> c_int; + + pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn abi_version_matches_the_loaded_library() { + assert_eq!( + unsafe { xl_abi_version() }, + XL_ABI_VERSION, + "the linked native library speaks a different ABI revision than this crate" + ); + } +} diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 64e81bc5..1d9dcb7e 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -1,515 +1,55 @@ -// Raw FFI bindings to ExcelReader's C ABI (`excelreader.h`). The `extern "C"` block below is -// declared directly in this crate root (not a submodule), so `crate::xl_open_file` etc. resolve -// from anywhere in the crate. See the `workbook` module for the safe wrapper built on top of this. #![doc = include_str!("../README.md")] #![doc( html_logo_url = "https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png", html_favicon_url = "https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png" )] -pub mod aggregate; +mod aggregate; mod error; +pub(crate) mod ffi; mod options; +mod rows; +mod source; mod temporal; -pub mod workbook; -pub mod writer; -pub mod writer_handle; -pub mod rows; -pub mod source; +mod workbook; +mod writer; +mod writer_handle; #[cfg(feature = "arrow")] -pub mod arrow; +mod arrow; -pub use aggregate::{aggregate_csv_file, aggregate_csv_memory, CsvAccumulator, CsvParallelOptions}; -pub use error::Error; -pub use rows::{AllRows, CellIter, CellRef, CellType, DecodedRows, RowCursor, RowRef}; +pub use aggregate::{aggregate_csv, CsvAccumulator, CsvInput}; +pub use error::{Error, ErrorKind}; +pub use options::{Format, OpenOptions, WriteOptions}; +pub use rows::{AllRows, CellIter, CellRef, CellType, RowCursor, RowRef}; pub use source::Source; -pub use options::{OpenOptions, OpenOptionsRaw, WriteOptions}; pub use temporal::{Date, Time, Timestamp}; +pub use workbook::{ + ColumnType, ExcelMapper, InferredColumn, Sheet, SheetVisibility, TableView, TableViewIter, + TypedChunks, Workbook, +}; +pub use writer::{ + encrypt_package, write_columns, write_columns_to, write_sheet, write_sheet_to, Column, + ColumnData, ExcelWriter, +}; +pub use writer_handle::WriterHandle; -use std::os::raw::{c_int, c_void}; - -pub const XL_OK: i32 = 0; -pub const XL_EOF: i32 = -1; -pub const XL_BUFFER_TOO_SMALL: i32 = -2; -pub const XL_INVALID_HANDLE: i32 = -3; -pub const XL_INVALID_ARGUMENT: i32 = -4; -pub const XL_ERROR: i32 = -5; -/// The workbook is encrypted and no password was supplied. -pub const XL_STATUS_PASSWORD_REQUIRED: i32 = -6; -/// The supplied password did not match the workbook's verifier. -pub const XL_STATUS_PASSWORD_INCORRECT: i32 = -7; - -pub const XL_CELL_EMPTY: i32 = 0; -pub const XL_CELL_STRING: i32 = 1; -pub const XL_CELL_NUMBER: i32 = 2; -pub const XL_CELL_DATE: i32 = 3; -pub const XL_CELL_BOOL: i32 = 4; -pub const XL_CELL_FORMULA: i32 = 5; -pub const XL_CELL_ERROR: i32 = 6; - -/// ABI revision this crate is compiled against. `Workbook::open` refuses to proceed when the loaded -/// library's `xl_abi_version()` disagrees - see `workbook::check_abi_version`. -pub const XL_ABI_VERSION: i32 = 6; - -pub const XL_T_STRING: i32 = 0; -pub const XL_T_I64: i32 = 1; -pub const XL_T_F64: i32 = 2; -pub const XL_T_BOOL: i32 = 3; -/// Days since 1970-01-01, stored as `i32`. -pub const XL_T_DATE: i32 = 4; -/// Microseconds since midnight, stored as `i64`. -pub const XL_T_TIME: i32 = 5; -/// Microseconds since 1970-01-01T00:00:00Z, stored as `i64`. -pub const XL_T_TIMESTAMP: i32 = 6; - -/// `xl_infer_schema` flag: also type cells that hold text (integers, decimals, booleans, ISO dates). -pub const XL_INFER_PARSE_TEXT: i32 = 1; - -pub const XL_SHEET_VISIBLE: i32 = 0; -pub const XL_SHEET_HIDDEN: i32 = 1; -pub const XL_SHEET_VERY_HIDDEN: i32 = 2; - -pub const XL_FORMAT_AUTO: i32 = 0; -pub const XL_FORMAT_XLS: i32 = 1; -pub const XL_FORMAT_XLSX: i32 = 2; -pub const XL_FORMAT_XLSB: i32 = 3; -pub const XL_FORMAT_CSV: i32 = 4; - -/// Every boolean-shaped `xl_open_options` field uses one of these three states, never a plain 0/1 - -/// several of them default to true, so a bare 0 would be ambiguous between "off" and "use the -/// library default". -pub const XL_OPT_DEFAULT: i32 = 0; -pub const XL_OPT_FALSE: i32 = 1; -pub const XL_OPT_TRUE: i32 = 2; - -/// Opaque handle - never dereferenced by Rust, only passed back to `xl_*` functions. -#[repr(C)] -pub struct XlWorkbook { - _private: [u8; 0], -} - -/// Opaque row cursor (`xl_rows`) - never dereferenced by Rust, only passed back to `xl_rows_*`. -#[repr(C)] -pub struct XlRows { - _private: [u8; 0], -} - -#[repr(C)] -pub struct XlTypedReader { - _private: [u8; 0], -} - -/// Mirrors `xl_open_options`. `struct_size` must be set to `size_of::()`; build one -/// through [`OpenOptions`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy, Debug)] -pub struct XlOpenOptions { - pub struct_size: i32, - pub csv_sniff_dialect: i32, - pub csv_delimiter: i32, - pub csv_quote: i32, - pub csv_detect_bom: i32, - pub csv_max_cell_bytes: i32, - pub csv_intern_strings: i32, - pub max_total_decompressed_bytes: i64, - pub max_cell_bytes: i32, - pub max_shared_string_bytes: i64, - pub max_zip_entries: i32, - pub prefetch_decompression: i32, - pub intern_strings: i32, - - /// Password for an encrypted OOXML workbook, as UTF-8 bytes. NULL (with `password_len` 0) means - /// "not encrypted, or fail with `XL_STATUS_PASSWORD_REQUIRED`". Not NUL-terminated - - /// `password_len` is authoritative, since a password may contain any byte. The pointer need only - /// remain valid for the duration of the call - see [`crate::options::OpenOptionsRaw`]. - pub password: *const u8, - pub password_len: i32, - pub source_cache_bytes: i64, - pub max_buffered_bytes: i64, - pub source_block_size: i32, - pub verify_zip_checksums: i32, -} - -#[repr(C)] -pub struct XlSource { - pub struct_size: i32, - pub user_data: *mut c_void, - pub length: i64, - pub read_at: Option i64>, - pub release: Option, -} - -#[repr(C)] -pub struct XlStream { - pub struct_size: i32, - pub user_data: *mut c_void, - pub read: Option i64>, - pub release: Option, -} - -/// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with -/// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts -/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. -/// Build one through [`WriteOptions`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy, Debug)] -pub struct XlWriteOptions { - pub struct_size: i32, - pub sheet_name_len: i32, - pub sheet_name: *const u8, - pub csv_delimiter: i32, - pub csv_quote: i32, - pub date1904: i32, - pub use_shared_strings: i32, -} - -#[repr(C)] -pub struct XlColumnSpec { - pub names: *const *const u8, - pub name_lens: *const i32, - pub name_count: i32, - pub index: i32, - pub r#type: i32, - pub nullable: i32, -} - -#[repr(C)] -pub struct XlColumn { - pub r#type: i32, - pub length: i64, - pub values: *const c_void, - pub validity: *const u8, - pub data: *const u8, - pub data_len: i64, -} - -#[repr(C)] -pub struct XlTable { - pub column_count: i32, - pub row_count: i64, - pub columns: *mut XlColumn, -} - -/// Mirrors `xl_inferred_schema`. Owned by the native library until `xl_free_schema`. -#[repr(C)] -pub struct XlInferredSchema { - pub columns: *mut XlColumnSpec, - pub column_count: i32, -} - -/// Mirrors `xl_buffer`: an owned block of native memory returned by `xl_write_typed_to_memory` or -/// `xl_write_handle_bytes`. Released with `xl_free_buffer` - see `writer::buffer_to_vec`, the one -/// place this crate touches the raw struct directly. -#[repr(C)] -pub struct XlBuffer { - pub data: *mut u8, - pub len: i64, -} - -/// Opaque streaming writer handle - never dereferenced by Rust, only passed back to `xl_*` -/// functions. See [`writer_handle::WriterHandle`] for the safe wrapper. -#[repr(C)] -pub struct XlWriterHandle { - _private: [u8; 0], -} - -/// Mirrors `xl_row_cell`. Named `cell_type` because `type` is a Rust keyword. -#[repr(C)] -#[derive(Debug)] -pub struct XlRowCell { - pub column: i32, - pub cell_type: i32, - pub value_len: i32, - pub value: *const u8, -} - -/// Mirrors `xl_row`. -#[repr(C)] -pub struct XlRow { - pub cell_count: i32, - pub cells: *mut XlRowCell, -} - -/// Mirrors `xl_rows_decoded`. -#[repr(C)] -pub struct XlRowsDecoded { - pub row_count: i32, - pub rows: *mut XlRow, -} - -/// Mirrors `xl_csv_aggregation`. Every callback runs on a library worker thread except -/// `free_state`, which runs on the calling thread after the run has finished; none may let a -/// panic escape into the library - build one through [`aggregate`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy)] -pub struct XlCsvAggregation { - pub struct_size: i32, - pub seed: Option i32>, - pub accumulate: Option i32>, - pub combine: Option i32>, - pub free_state: Option, - pub user_data: *mut c_void, -} - -/// Mirrors `xl_csv_parallel_options`. `struct_size` must be set to -/// `size_of::()`; build one through [`CsvParallelOptions`] rather than by hand. -#[repr(C)] -#[derive(Clone, Copy, Debug)] -pub struct XlCsvParallelOptions { - pub struct_size: i32, - pub degree_of_parallelism: i32, - pub header_row: i32, - pub delimiter: i32, - pub quote: i32, - pub detect_bom: i32, - pub max_cell_bytes: i32, -} - -extern "C" { - pub fn xl_abi_version() -> c_int; - - pub fn xl_open_file( - path: *const u8, - path_len: i32, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_open_memory( - data: *const u8, - data_len: i32, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_open_source( - source: *const XlSource, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_open_stream( - stream: *const XlStream, - format: i32, - options: *const XlOpenOptions, - out_handle: *mut *mut XlWorkbook, - ) -> c_int; - - pub fn xl_set_source_error(message: *const u8, len: i32); - - pub fn xl_close(handle: *mut XlWorkbook) -> c_int; - - pub fn xl_sheet_count(handle: *mut XlWorkbook, out_count: *mut i32) -> c_int; - - pub fn xl_sheet_name_at( - handle: *mut XlWorkbook, - index: i32, - buffer: *mut u8, - capacity: i32, - out_len: *mut i32, - ) -> c_int; - - pub fn xl_sheet_visibility_at( - handle: *mut XlWorkbook, - index: i32, - out_visibility: *mut i32, - ) -> c_int; - - /// A name that matches no sheet is `XL_OK` with `-1`. - pub fn xl_sheet_index( - handle: *mut XlWorkbook, - name: *const u8, - name_len: i32, - out_index: *mut i32, - ) -> c_int; - - pub fn xl_is_date1904(handle: *mut XlWorkbook, out_flag: *mut i32) -> c_int; - - pub fn xl_parse_typed( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - degree_of_parallelism: i32, - out_table: *mut XlTable, - ) -> c_int; - - /// Same schema-driven parse as `xl_parse_typed`, exported as one top-level Arrow struct - /// array/schema. `out_array`/`out_schema` are `struct ArrowArray*`/`struct ArrowSchema*` from - /// the Arrow C Data Interface, typed here as `c_void` because arrow-rs's own `#[repr(C)]` - /// `FFI_ArrowArray`/`FFI_ArrowSchema` are ABI-identical to them - redeclaring the spec structs - /// would be a second source of truth for a fixed, versioned ABI. - /// - /// On `XL_OK` the caller owns both and releases each through its OWN `release` callback, never - /// through `xl_free_table`. On any other status both outputs are left untouched. - pub fn xl_parse_arrow( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - degree_of_parallelism: i32, - out_array: *mut c_void, - out_schema: *mut c_void, - ) -> c_int; - - /// `out_stream` is a `struct ArrowArrayStream*`, typed as `c_void` for the same reason - /// `xl_parse_arrow`'s outputs are. Its `release` is what closes the underlying read. - pub fn xl_parse_arrow_stream( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - max_rows: i64, - out_stream: *mut c_void, - ) -> c_int; - - pub fn xl_free_table(table: *mut XlTable); - - /// `max_rows` is rows per batch, 0 meaning one unbounded batch. - pub fn xl_typed_reader_open( - handle: *mut XlWorkbook, - sheet: i32, - specs: *const XlColumnSpec, - spec_count: i32, - header_row: i32, - max_rows: i64, - out_reader: *mut *mut XlTypedReader, - ) -> c_int; - - /// `XL_OK` with a batch, or `XL_EOF` at end of sheet. A failure latches: every later call - /// repeats it. - pub fn xl_typed_reader_next(reader: *mut XlTypedReader, out_table: *mut XlTable) -> c_int; - - pub fn xl_typed_reader_close(reader: *mut XlTypedReader); - - pub fn xl_infer_schema( - handle: *mut XlWorkbook, - sheet: i32, - header_row: i32, - sample_size: i32, - flags: i32, - out_schema: *mut XlInferredSchema, - ) -> c_int; - - pub fn xl_free_schema(schema: *mut XlInferredSchema); - pub fn xl_write_typed( - path: *const u8, - path_len: i32, - format: i32, - specs: *const XlColumnSpec, - table: *const XlTable, - options: *const XlWriteOptions, - ) -> c_int; - - /// Same as `xl_write_typed`, except the result is returned as `out_buffer` instead of being - /// written to a path. Only read `*out_buffer` when the call returns `XL_OK`; on failure it is - /// zeroed, and `xl_free_buffer` on a zeroed buffer is a no-op. - pub fn xl_write_typed_to_memory( - format: i32, - specs: *const XlColumnSpec, - table: *const XlTable, - options: *const XlWriteOptions, - out_buffer: *mut XlBuffer, - ) -> c_int; - - /// Releases a buffer returned by `xl_write_typed_to_memory` or `xl_write_handle_bytes` and - /// resets it to zero. Safe on a zeroed value. - pub fn xl_free_buffer(buffer: *mut XlBuffer); - - /// Reads the plaintext XLSX/XLSB package at `package_path` and writes its agile-encrypted - /// (ECMA-376 4.4) counterpart to `destination_path`, overwriting an existing file. The result - /// opens with the same password via `xl_open_file`'s `xl_open_options::password`. - pub fn xl_encrypt_package( - package_path: *const u8, - package_path_len: i32, - destination_path: *const u8, - destination_path_len: i32, - password: *const u8, - password_len: i32, - ) -> c_int; - - - pub fn xl_open_write_handle( - path: *const u8, - path_len: i32, - format: i32, - options: *const XlWriteOptions, - out_handle: *mut *mut XlWriterHandle, - ) -> c_int; - - pub fn xl_open_write_handle_to_memory( - format: i32, - options: *const XlWriteOptions, - out_handle: *mut *mut XlWriterHandle, - ) -> c_int; - - pub fn xl_start_sheet(handle: *mut XlWriterHandle, name: *const u8, name_len: i32) -> c_int; - pub fn xl_start_row(handle: *mut XlWriterHandle) -> c_int; - - pub fn xl_write_string(handle: *mut XlWriterHandle, value: *const u8, value_len: i32) -> c_int; - pub fn xl_write_int64(handle: *mut XlWriterHandle, value: i64) -> c_int; - pub fn xl_write_float64(handle: *mut XlWriterHandle, value: f64) -> c_int; - pub fn xl_write_bool(handle: *mut XlWriterHandle, value: i32) -> c_int; - pub fn xl_write_date(handle: *mut XlWriterHandle, days_since_epoch: i32) -> c_int; - pub fn xl_write_time(handle: *mut XlWriterHandle, micros_since_midnight: i64) -> c_int; - pub fn xl_write_timestamp(handle: *mut XlWriterHandle, micros_since_epoch: i64) -> c_int; - pub fn xl_write_null(handle: *mut XlWriterHandle, r#type: i32) -> c_int; - - pub fn xl_end_row(handle: *mut XlWriterHandle) -> c_int; - pub fn xl_end_sheet(handle: *mut XlWriterHandle) -> c_int; - pub fn xl_close_write_handle(handle: *mut XlWriterHandle) -> c_int; - - /// Reads back everything written so far to a handle opened by `xl_open_write_handle_to_memory`. - /// `XL_INVALID_ARGUMENT` for one from `xl_open_write_handle`. Only read `*out_buffer` when the - /// call returns `XL_OK`; on failure it is zeroed, and `xl_free_buffer` on a zeroed buffer is a - /// no-op. - pub fn xl_write_handle_bytes(handle: *mut XlWriterHandle, out_buffer: *mut XlBuffer) -> c_int; - - pub fn xl_rows_open(handle: *mut XlWorkbook, sheet: i32, out_rows: *mut *mut XlRows) -> c_int; - pub fn xl_rows_close(rows: *mut XlRows) -> c_int; - pub fn xl_rows_next( - rows: *mut XlRows, - buffer: *mut u8, - capacity: i32, - out_written: *mut i32, - ) -> c_int; - pub fn xl_rows_read_all_blob( - rows: *mut XlRows, - buffer: *mut u8, - capacity: i32, - out_written: *mut i32, - ) -> c_int; - pub fn xl_rows_read_all_decoded(rows: *mut XlRows, out_rows: *mut XlRowsDecoded) -> c_int; - pub fn xl_free_rows(rows: *mut XlRowsDecoded); - - /// Folds a CSV file into one caller-owned accumulator across several threads. Blocks until the - /// run finishes; the surviving state is written to `out_state` only on `XL_OK` and becomes the - /// caller's to free. See the contract above `xl_csv_aggregation` in `excelreader.h`, and - /// [`aggregate::aggregate_csv_file`] for the safe wrapper. - pub fn xl_csv_aggregate_file( - path: *const u8, - path_len: i32, - agg: *const XlCsvAggregation, - options: *const XlCsvParallelOptions, - out_state: *mut *mut c_void, - ) -> c_int; - - /// Same as `xl_csv_aggregate_file` over an in-memory buffer, which must stay valid and - /// unmodified until the call returns - the library does not copy it. - pub fn xl_csv_aggregate_memory( - data: *const u8, - data_len: i32, - agg: *const XlCsvAggregation, - options: *const XlCsvParallelOptions, - out_state: *mut *mut c_void, - ) -> c_int; - - pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; +#[cfg(feature = "arrow")] +pub use self::arrow::ArrowChunks; + +/// Support for `#[derive(ExcelMapper)]`'s generated code. Not part of the public API: nothing here +/// follows semver. +#[doc(hidden)] +pub mod __private { + pub use crate::ffi::XlColumn; + pub use crate::workbook::{ + column_bool, column_date, column_f64, column_i64, column_str, column_time, + column_timestamp, ColumnBinding, + }; + pub use crate::writer::{push_str, set_valid, OwnedColumn, OwnedColumnData}; + + #[must_use] + pub fn invalid_argument(message: String) -> crate::Error { + crate::Error::new(crate::ErrorKind::InvalidArgument, message) + } } diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 8f9342ec..27e0f31b 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -1,56 +1,100 @@ -//! Builder for [`XlOpenOptions`], mirroring the Python binding's `OpenOptions` and the C++ -//! wrapper's `xl::OpenOptions`. +//! Builders for the ABI's open, write and CSV-aggregation option structs, mirroring the Python +//! binding's `OpenOptions` and the C++ wrapper's `xl::OpenOptions`. //! //! The ABI splits "unset" across two conventions: `0` for a numeric field, `XL_OPT_DEFAULT` for a -//! boolean-shaped one. This type presents the single `None` convention the other bindings expose and -//! does the split in [`OpenOptions::to_raw`]. +//! boolean-shaped one. These types present the single `None` convention the other bindings expose +//! and do the split when lowering. //! //! Values are passed through unvalidated on purpose: the native side owns the real bounds and //! reports a rejection through `xl_last_error`, so checking them here too would give those bounds a //! second place to drift from. -use crate::{XlOpenOptions, XlWriteOptions, XL_OPT_DEFAULT, XL_OPT_FALSE, XL_OPT_TRUE}; +use std::path::Path; + +use crate::ffi::{ + self, XlCsvParallelOptions, XlOpenOptions, XlWriteOptions, XL_OPT_DEFAULT, XL_OPT_FALSE, + XL_OPT_TRUE, +}; +use crate::{Error, ErrorKind}; + +/// A workbook file format. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Format { + /// Opening: sniff the format from the file's signature bytes. Sniffing does not detect CSV. + /// Keep `Auto` for a password-protected workbook: an explicit `Xlsx`/`Xlsb` reads its + /// encryption wrapper as a plain ZIP and fails. Writing: infer the format from the path's + /// extension. + #[default] + Auto, + Xls, + Xlsx, + Xlsb, + Csv, +} + +impl Format { + pub(crate) fn raw(self) -> i32 { + match self { + Format::Auto => ffi::XL_FORMAT_AUTO, + Format::Xls => ffi::XL_FORMAT_XLS, + Format::Xlsx => ffi::XL_FORMAT_XLSX, + Format::Xlsb => ffi::XL_FORMAT_XLSB, + Format::Csv => ffi::XL_FORMAT_CSV, + } + } + + /// Infers a format from a path's extension, case-insensitively. `Auto` when the extension is + /// absent or unrecognized - which the native write then rejects, since a file being created has + /// no signature bytes to sniff. + fn from_path(path: &Path) -> Format { + let extension = path.extension().and_then(|e| e.to_str()).unwrap_or_default(); + match extension.to_ascii_lowercase().as_str() { + "xlsx" => Format::Xlsx, + "xlsb" => Format::Xlsb, + "xls" => Format::Xls, + "csv" => Format::Csv, + _ => Format::Auto, + } + } +} -/// Options for [`Workbook::open_with`](crate::workbook::Workbook::open_with) and -/// [`Workbook::open_memory`](crate::workbook::Workbook::open_memory). Every field is `None` by -/// default, meaning "use the library default"; set only the ones you want to override. +/// How to open a workbook, in the style of [`std::fs::OpenOptions`]: build one, then call +/// [`open`](Self::open), [`open_bytes`](Self::open_bytes), [`open_source`](Self::open_source) or +/// [`open_reader`](Self::open_reader). Every setting is unset by default, meaning "use the library +/// default"; set only the ones you want to override. /// /// ```no_run -/// use excelreader::OpenOptions; +/// use excelreader::{Format, OpenOptions}; /// -/// let options = OpenOptions::new() -/// .prefetch_decompression(true) -/// .max_zip_entries(1024); +/// let workbook = OpenOptions::new() +/// .format(Format::Csv) +/// .csv_delimiter(b';' as i32) +/// .open("data.csv")?; +/// # Ok::<(), excelreader::Error>(()) /// ``` #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct OpenOptions { - pub csv_sniff_dialect: Option, - pub csv_delimiter: Option, - pub csv_quote: Option, - pub csv_detect_bom: Option, - pub csv_max_cell_bytes: Option, - pub csv_intern_strings: Option, - - pub max_total_decompressed_bytes: Option, - pub max_cell_bytes: Option, - pub max_shared_string_bytes: Option, - pub max_zip_entries: Option, - pub prefetch_decompression: Option, - pub intern_strings: Option, - /// XLSX/XLSB only: check each ZIP entry against its CRC-32. `None` is `true`. - pub verify_zip_checksums: Option, - - /// `open_source` only: bytes fetched per `read_at` call. `None` is 4 MiB; a negative value turns - /// the cache off. - pub source_block_size: Option, - /// `open_source` only: bytes of blocks kept in memory. `None` is 64 MiB. - pub source_cache_bytes: Option, - /// `open_reader` with XLSX, XLSB or XLS only: the most bytes the reader may hold. - pub max_buffered_bytes: Option, - - /// Password for an encrypted OOXML workbook. `None` means "not encrypted, or fail with - /// `Error::PasswordRequired`". - pub password: Option, + format: Format, + + csv_sniff_dialect: Option, + csv_delimiter: Option, + csv_quote: Option, + csv_detect_bom: Option, + + max_total_decompressed_bytes: Option, + max_cell_bytes: Option, + max_shared_string_bytes: Option, + max_zip_entries: Option, + prefetch_decompression: Option, + intern_strings: Option, + verify_zip_checksums: Option, + + source_block_size: Option, + source_cache_bytes: Option, + max_buffered_bytes: Option, + + password: Option, } /// Generates a consuming builder setter per field, so callers can chain overrides. @@ -68,12 +112,20 @@ macro_rules! setters { } impl OpenOptions { - /// Every field unset - identical to passing no options at all. + /// Every setting unset: sniff the format, use every library default. #[must_use] pub fn new() -> Self { Self::default() } + /// The file format. Default [`Format::Auto`], which does not detect CSV - set + /// [`Format::Csv`] to open one. + #[must_use] + pub fn format(mut self, value: Format) -> Self { + self.format = value; + self + } + setters! { /// CSV only. Infers delimiter/quote/encoding from a leading sample before opening, which /// makes `csv_delimiter`/`csv_quote` ignored. @@ -84,13 +136,9 @@ impl OpenOptions { csv_quote: i32, /// CSV only. Library default is `true`. csv_detect_bom: bool, - /// CSV only. Library default is 32 MiB. - csv_max_cell_bytes: i32, - /// CSV only. Library default is `false`. - csv_intern_strings: bool, /// XLS/XLSX/XLSB only. Library default is 512 MiB. max_total_decompressed_bytes: i64, - /// XLS/XLSX/XLSB only. Library default is 32 MiB. + /// Largest cell value, or buffered CSV record. Library default is 32 MiB. max_cell_bytes: i32, /// XLS/XLSX/XLSB only. Library default is 128 MiB. max_shared_string_bytes: i64, @@ -98,41 +146,45 @@ impl OpenOptions { max_zip_entries: i32, /// XLS/XLSX/XLSB only. Library default is `false`. prefetch_decompression: bool, - /// XLS/XLSX/XLSB only. Library default is `false`. + /// Deduplicate repeated cell strings. Library default is `false`. intern_strings: bool, /// XLSX/XLSB only: check each ZIP entry against its CRC-32, so a damaged file fails the read /// instead of returning wrong cells. Library default is `true`. verify_zip_checksums: bool, - /// `open_source` only: bytes fetched per `read_at` call. Library default is 4 MiB; a negative - /// value turns the cache off. + /// [`open_source`](Self::open_source) only: bytes fetched per `read_at` call. Library + /// default is 4 MiB; a negative value turns the cache off. source_block_size: i32, - /// `open_source` only: bytes of blocks kept in memory. Library default is 64 MiB. + /// [`open_source`](Self::open_source) only: bytes of blocks kept in memory. Library + /// default is 64 MiB. source_cache_bytes: i64, - /// `open_reader` with XLSX, XLSB or XLS only: the most bytes the reader may hold. + /// [`open_reader`](Self::open_reader) with XLSX, XLSB or XLS only: the most bytes the + /// reader may hold. max_buffered_bytes: i64, } - /// Password for an encrypted OOXML workbook. Stored owned, since the C ABI receives a pointer - /// that must stay valid for the duration of the open call. - /// - /// Written by hand rather than generated by `setters!`: this is the one field that is not a - /// plain `Copy` scalar. + /// Password for an encrypted OOXML workbook. Unset means "not encrypted, or fail with + /// [`ErrorKind::PasswordRequired`]". #[must_use] pub fn password(mut self, value: impl Into) -> Self { self.password = Some(value.into()); self } + pub(crate) fn format_raw(&self) -> i32 { + self.format.raw() + } + /// Lowers this into the raw ABI struct, with `struct_size` filled in. /// /// Returns a guard rather than a bare `XlOpenOptions`: when [`password`](Self::password) is /// set, the raw struct's `password` field is a pointer into `self.password`'s bytes, which must - /// outlive every use of the pointer. Returning `XlOpenOptions` by value would let that pointer - /// dangle the moment the return value's producer (this call) went out of scope - the borrow on - /// [`OpenOptionsRaw`] below is what makes the compiler refuse a call site that does not keep - /// `self` alive for as long as the raw struct is in use. - #[must_use] - pub fn to_raw(&self) -> OpenOptionsRaw<'_> { + /// outlive every use of the pointer. The borrow on [`OpenOptionsRaw`] is what makes the + /// compiler refuse a call site that does not keep `self` alive for as long as the raw struct is + /// in use. + /// + /// `max_cell_bytes`/`intern_strings` lower into both the general and the `csv_*` fields: the + /// ABI reads the `csv_*` pair for CSV and the other pair for every other format. + pub(crate) fn to_raw(&self) -> OpenOptionsRaw<'_> { let password = self.password.as_deref().map(str::as_bytes); let mut raw = XlOpenOptions { struct_size: std::mem::size_of::() as i32, @@ -140,8 +192,8 @@ impl OpenOptions { csv_delimiter: opt_number(self.csv_delimiter), csv_quote: opt_number(self.csv_quote), csv_detect_bom: opt_state(self.csv_detect_bom), - csv_max_cell_bytes: opt_number(self.csv_max_cell_bytes), - csv_intern_strings: opt_state(self.csv_intern_strings), + csv_max_cell_bytes: opt_number(self.max_cell_bytes), + csv_intern_strings: opt_state(self.intern_strings), max_total_decompressed_bytes: opt_number(self.max_total_decompressed_bytes), max_cell_bytes: opt_number(self.max_cell_bytes), max_shared_string_bytes: opt_number(self.max_shared_string_bytes), @@ -164,25 +216,36 @@ impl OpenOptions { _password: password, } } + + /// The CSV dialect and cell limit of these options, plus the two per-call settings, as the + /// aggregation ABI takes them. + pub(crate) fn to_csv_parallel_raw( + &self, + header_row: i32, + degree_of_parallelism: i32, + ) -> XlCsvParallelOptions { + XlCsvParallelOptions { + struct_size: std::mem::size_of::() as i32, + degree_of_parallelism, + header_row, + delimiter: opt_number(self.csv_delimiter), + quote: opt_number(self.csv_quote), + detect_bom: opt_state(self.csv_detect_bom), + max_cell_bytes: opt_number(self.max_cell_bytes), + } + } } /// Holds the raw C struct together with the buffer its `password` pointer refers to. The raw struct /// must never outlive this guard, and the borrow of `OpenOptions` (carried through `'a`) is what -/// guarantees it: `_password` borrows the same `OpenOptions::password` string `raw.password` points -/// into, so the borrow checker keeps this guard's lifetime from exceeding `self`'s. -/// -/// `_password` is never read - it exists purely to hold that borrow. Dropping it early would not -/// itself free anything (the bytes live in `OpenOptions`, not here), but nothing stops a future edit -/// from reordering fields in a way that matters, so keep it paired with `raw` for the same reason -/// `SpecArena` in `workbook.rs` keeps its own otherwise-unread borrow fields. -pub struct OpenOptionsRaw<'a> { +/// guarantees it. +pub(crate) struct OpenOptionsRaw<'a> { raw: XlOpenOptions, _password: Option<&'a [u8]>, } impl OpenOptionsRaw<'_> { - /// Pointer to the raw struct, valid for as long as this guard is alive (and no longer - the - /// password bytes it may point into are borrowed from the `OpenOptions` this guard came from). + /// Pointer to the raw struct, valid for as long as this guard is alive. pub(crate) fn as_ptr(&self) -> *const XlOpenOptions { &self.raw } @@ -203,8 +266,8 @@ fn opt_number(value: Option) -> T { value.unwrap_or_default() } -/// Options for [`write_columns`](crate::writer::write_columns) and -/// [`write_sheet`](crate::writer::write_sheet). Every field is `None` by default, meaning "use the +/// Options for [`write_columns`](crate::write_columns), [`write_sheet`](crate::write_sheet) and +/// [`WriterHandle`](crate::WriterHandle). Every setting is unset by default, meaning "use the /// library default"; set only the ones you want to override. /// /// ```no_run @@ -214,29 +277,31 @@ fn opt_number(value: Option) -> T { /// ``` #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct WriteOptions { - /// Sheet name. `None` = `"Sheet1"`. Ignored for CSV. Excel's rules (1-31 characters, none of - /// `: \ / ? * [ ]`) are enforced natively and reported through `xl_last_error`. - pub sheet_name: Option, - - pub csv_delimiter: Option, - pub csv_quote: Option, - - /// XLS/XLSB only; ignored for XLSX and CSV. Library default is `false`. - pub date1904: Option, - /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at the - /// cost of a string table. Library default is `false`. - pub use_shared_strings: Option, + format: Format, + sheet_name: Option, + csv_delimiter: Option, + csv_quote: Option, + date1904: Option, + use_shared_strings: Option, } impl WriteOptions { - /// Every field unset - identical to passing no options at all. + /// Every setting unset. #[must_use] pub fn new() -> Self { Self::default() } - /// Written by hand rather than generated by `setters!`: this is the one field that is not a - /// plain `Copy` scalar. + /// The output format. Default [`Format::Auto`]: inferred from the path's extension + /// (`.xlsx`/`.xlsb`/`.xls`/`.csv`). Writing to an `impl Write` needs an explicit format. + #[must_use] + pub fn format(mut self, value: Format) -> Self { + self.format = value; + self + } + + /// Sheet name. Unset is `"Sheet1"`. Ignored for CSV. Excel's rules (1-31 characters, none of + /// `: \ / ? * [ ]`) are enforced natively. #[must_use] pub fn sheet_name(mut self, value: impl Into) -> Self { self.sheet_name = Some(value.into()); @@ -248,18 +313,28 @@ impl WriteOptions { csv_delimiter: i32, /// CSV only. Byte value 1-255; the library default is `"`. csv_quote: i32, - /// XLS/XLSB only. Library default is `false`. + /// XLS/XLSB only; ignored for XLSX and CSV. Library default is `false`. date1904: bool, - /// XLSX/XLSB only. Library default is `false`. + /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at + /// the cost of a string table. Library default is `false`. use_shared_strings: bool, } - /// Lowers this into the raw ABI struct, with `struct_size` filled in. - /// - /// `pub(crate)`, unlike [`OpenOptions::to_raw`], which is public: the struct this returns holds - /// a raw pointer INTO `self.sheet_name`, so handing it to a caller who can outlive `self` is a - /// use-after-free waiting to happen. `write_columns` takes `Option<&WriteOptions>` and does the - /// lowering itself, where the borrow provably covers the FFI call. + /// The `XL_FORMAT_*` to write: the explicit format, else one inferred from `path`. With no path + /// to infer from, `Auto` is rejected here rather than by the native side. + pub(crate) fn format_for(&self, path: Option<&Path>) -> Result { + match (self.format, path) { + (Format::Auto, Some(path)) => Ok(Format::from_path(path).raw()), + (Format::Auto, None) => Err(Error::new( + ErrorKind::InvalidArgument, + "writing to an io::Write needs an explicit WriteOptions::format", + )), + (format, _) => Ok(format.raw()), + } + } + + /// Lowers this into the raw ABI struct, with `struct_size` filled in. The struct holds a raw + /// pointer INTO `self.sheet_name`, so it must not outlive `self`. pub(crate) fn to_raw(&self) -> XlWriteOptions { XlWriteOptions { struct_size: std::mem::size_of::() as i32, @@ -276,15 +351,6 @@ impl WriteOptions { } } -/// The pointer an FFI options argument takes: a borrowed raw struct, or NULL for "every library -/// default". `raw` must outlive the call the returned pointer is handed to - taking it by reference -/// rather than by value is what makes the compiler check that. -#[inline] -pub(crate) fn ptr_or_null(raw: &Option) -> *const T { - raw.as_ref() - .map_or(std::ptr::null(), |value| value as *const T) -} - #[cfg(test)] mod tests { use super::*; @@ -304,6 +370,7 @@ mod tests { "no password set means a null pointer" ); assert_eq!(raw.password_len, 0); + assert_eq!(options.format_raw(), ffi::XL_FORMAT_AUTO); } #[test] @@ -333,6 +400,33 @@ mod tests { assert_eq!(raw.max_shared_string_bytes, 1 << 20); } + #[test] + fn cell_limit_and_interning_reach_both_the_csv_and_the_general_fields() { + let options = OpenOptions::new().max_cell_bytes(4096).intern_strings(true); + let guard = options.to_raw(); + let raw = guard.raw; + assert_eq!(raw.max_cell_bytes, 4096); + assert_eq!(raw.csv_max_cell_bytes, 4096); + assert_eq!(raw.intern_strings, XL_OPT_TRUE); + assert_eq!(raw.csv_intern_strings, XL_OPT_TRUE); + } + + #[test] + fn csv_aggregation_options_take_the_dialect_and_cell_limit() { + let raw = OpenOptions::new() + .csv_delimiter(b';' as i32) + .csv_detect_bom(false) + .max_cell_bytes(1024) + .to_csv_parallel_raw(1, 4); + assert_eq!(raw.struct_size, std::mem::size_of::() as i32); + assert_eq!(raw.header_row, 1); + assert_eq!(raw.degree_of_parallelism, 4); + assert_eq!(raw.delimiter, b';' as i32); + assert_eq!(raw.quote, 0); + assert_eq!(raw.detect_bom, XL_OPT_FALSE); + assert_eq!(raw.max_cell_bytes, 1024); + } + #[test] fn password_lowers_to_a_pointer_and_length_into_the_owning_options() { let options = OpenOptions::new().password("hunter2"); @@ -344,11 +438,8 @@ mod tests { assert_eq!(bytes, b"hunter2"); } - /// The regression this task exists to prevent: `to_raw` must not compile down to something that - /// lets the password pointer dangle when the source string is a temporary rather than a local - /// the caller happens to keep alive. `to_raw`'s borrow of `options` (carried by `OpenOptionsRaw`) - /// is what makes that impossible - `options` here is a local binding that outlives `guard`, even - /// though the `String` handed to `.password()` itself was a temporary. + /// `to_raw` must not compile down to something that lets the password pointer dangle when the + /// source string is a temporary rather than a local the caller happens to keep alive. #[test] fn password_survives_a_temporary_source() { let options = OpenOptions::new().password(String::from("hunter2")); @@ -361,10 +452,7 @@ mod tests { #[test] fn write_options_lower_to_the_abi_defaults() { let raw = WriteOptions::new().to_raw(); - assert_eq!( - raw.struct_size, - std::mem::size_of::() as i32 - ); + assert_eq!(raw.struct_size, std::mem::size_of::() as i32); assert!(raw.sheet_name.is_null(), "an unset sheet name means Sheet1"); assert_eq!(raw.sheet_name_len, 0); assert_eq!(raw.csv_delimiter, 0); @@ -386,17 +474,37 @@ mod tests { assert_eq!(raw.use_shared_strings, XL_OPT_TRUE); } + #[test] + fn write_format_is_explicit_or_inferred_from_the_extension() { + let auto = WriteOptions::new(); + let format = |path: &str| auto.format_for(Some(Path::new(path))).unwrap(); + assert_eq!(format("out.xlsx"), ffi::XL_FORMAT_XLSX); + assert_eq!(format("out.XLSB"), ffi::XL_FORMAT_XLSB); + assert_eq!(format("out.xls"), ffi::XL_FORMAT_XLS); + assert_eq!(format("out.csv"), ffi::XL_FORMAT_CSV); + assert_eq!(format("out.txt"), ffi::XL_FORMAT_AUTO); + assert_eq!(format("out"), ffi::XL_FORMAT_AUTO); + assert_eq!(format("v1.2/report"), ffi::XL_FORMAT_AUTO); + + let error = auto.format_for(None).expect_err("a writer has no extension to infer from"); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + + let csv = WriteOptions::new().format(Format::Csv); + assert_eq!(csv.format_for(Some(Path::new("out.xlsx"))).unwrap(), ffi::XL_FORMAT_CSV); + assert_eq!(csv.format_for(None).unwrap(), ffi::XL_FORMAT_CSV); + } + /// The C struct's x64 layout is pinned by XL_STATIC_ASSERT in - /// tests/ExcelReader.NativeSmoke/smoke.c (lines 66-73). A repr(C) struct that disagrees would - /// hand the native side garbage where it expects a length and a pointer. + /// tests/ExcelReader.NativeSmoke/smoke.c. A repr(C) struct that disagrees would hand the native + /// side garbage where it expects a length and a pointer. #[test] fn write_options_match_the_c_struct_size() { - assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 32); } #[test] fn source_layouts_match_the_c_structs() { - use crate::{XlOpenOptions, XlSource, XlStream}; + use crate::ffi::{XlSource, XlStream}; assert_eq!(std::mem::size_of::(), 112); assert_eq!(std::mem::offset_of!(XlOpenOptions, source_cache_bytes), 88); assert_eq!(std::mem::offset_of!(XlOpenOptions, max_buffered_bytes), 96); diff --git a/rust/excelreader/src/rows.rs b/rust/excelreader/src/rows.rs index c516de9a..af7eb2ee 100644 --- a/rust/excelreader/src/rows.rs +++ b/rust/excelreader/src/rows.rs @@ -1,9 +1,10 @@ //! Row-at-a-time and whole-sheet decoded reads over the C ABI's row APIs. -use crate::{ - Error, XlRowCell, XL_CELL_BOOL, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR, XL_CELL_FORMULA, - XL_CELL_NUMBER, XL_CELL_STRING, XL_ERROR, +use crate::ffi::{ + self, XlRowCell, XL_CELL_BOOL, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR, XL_CELL_FORMULA, + XL_CELL_NUMBER, XL_CELL_STRING, }; +use crate::{Error, ErrorKind}; /// The kind of a cell, mirroring `XL_CELL_*` in the C ABI. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -21,8 +22,7 @@ pub enum CellType { impl CellType { /// Maps a raw `XL_CELL_*` value. `None` for a value this crate does not know, which can only /// happen against a newer native library than the one this crate was built against. - #[must_use] - pub fn from_raw(value: i32) -> Option { + pub(crate) fn from_raw(value: i32) -> Option { match value { XL_CELL_EMPTY => Some(CellType::Empty), XL_CELL_STRING => Some(CellType::String), @@ -55,12 +55,12 @@ impl<'a> CellRef<'a> { /// workbook never produces. pub fn as_str(&self) -> Result<&'a str, Error> { std::str::from_utf8(self.value) - .map_err(|err| Error::from_status(XL_ERROR, format!("cell value is not valid UTF-8: {err}"))) + .map_err(|err| Error::new(ErrorKind::Other, format!("cell value is not valid UTF-8: {err}"))) } } -/// Where a `RowRef`'s cells live. Blob rows come from `xl_rows_next`, decoded rows from -/// `xl_rows_read_all_decoded`. +/// Where a `RowRef`'s cells live. Blob rows come from `xl_rows_next`/`xl_rows_read_all_blob`, +/// decoded rows from a CSV aggregation callback. #[derive(Debug, Clone, Copy)] enum RowBacking<'a> { /// The bytes AFTER the leading `int32 cell_count`. @@ -93,7 +93,7 @@ impl<'a> RowRef<'a> { /// /// # Safety /// `cells` must point to `count` initialized `XlRowCell` values whose `value` pointers stay - /// valid for `'a` — that is, until `xl_free_rows` releases the enclosing `XlRowsDecoded`. + /// valid for `'a`. pub(crate) unsafe fn from_decoded(cells: *const XlRowCell, count: i32) -> RowRef<'a> { let len = if count > 0 { count as usize } else { 0 }; let slice = if len == 0 || cells.is_null() { @@ -114,9 +114,8 @@ impl<'a> RowRef<'a> { self.len == 0 } - /// The cell at `index`. For a row from `RowCursor` this walks the blob from the start, so it is - /// O(index); prefer `iter()` when reading a whole row. A row from `DecodedRows` indexes - /// directly. + /// The cell at `index`. For a row read from a sheet this walks the row from the start, so it is + /// O(index); prefer `iter()` when reading a whole row. #[must_use] pub fn get(&self, index: usize) -> Option> { if index >= self.len { @@ -222,19 +221,19 @@ const INITIAL_ROW_BUFFER: usize = 64 * 1024; /// Not an `Iterator`: each row borrows the buffer that the next call overwrites, which /// `Iterator::next` cannot express. One row is alive at a time, enforced by the borrow checker. pub struct RowCursor<'w> { - rows: *mut crate::XlRows, + rows: *mut ffi::XlRows, buffer: Vec, written: usize, - workbook: std::marker::PhantomData<&'w crate::workbook::Workbook>, + workbook: std::marker::PhantomData<&'w crate::Workbook>, } // SAFETY: the cursor owns its native enumerator and ABI v6 lets one thread at a time use it; unsafe impl Send for RowCursor<'_> {} impl<'w> RowCursor<'w> { - pub(crate) fn open(handle: *mut crate::XlWorkbook, sheet: i32) -> Result, Error> { - let mut rows: *mut crate::XlRows = std::ptr::null_mut(); - crate::workbook::check(unsafe { crate::xl_rows_open(handle, sheet, &mut rows) })?; + pub(crate) fn open(handle: *mut ffi::XlWorkbook, sheet: i32) -> Result, Error> { + let mut rows: *mut ffi::XlRows = std::ptr::null_mut(); + crate::workbook::check(unsafe { ffi::xl_rows_open(handle, sheet, &mut rows) })?; Ok(RowCursor { rows, buffer: vec![0; INITIAL_ROW_BUFFER], @@ -243,7 +242,7 @@ impl<'w> RowCursor<'w> { }) } - pub(crate) fn raw(&self) -> *mut crate::XlRows { + pub(crate) fn raw(&self) -> *mut ffi::XlRows { self.rows } @@ -256,26 +255,26 @@ impl<'w> RowCursor<'w> { let mut written: i32 = 0; let capacity = i32::try_from(self.buffer.len()).unwrap_or(i32::MAX); let status = unsafe { - crate::xl_rows_next(self.rows, self.buffer.as_mut_ptr(), capacity, &mut written) + ffi::xl_rows_next(self.rows, self.buffer.as_mut_ptr(), capacity, &mut written) }; match status { - crate::XL_OK => { + ffi::XL_OK => { self.written = if written > 0 { written as usize } else { 0 }; let blob = &self.buffer[..self.written]; return Some( RowRef::from_blob(blob).ok_or_else(|| { - Error::from_status(XL_ERROR, "native returned a malformed row blob".to_string()) + Error::new(ErrorKind::Other, "native returned a malformed row blob") }), ); } - crate::XL_EOF => return None, - crate::XL_BUFFER_TOO_SMALL => { + ffi::XL_EOF => return None, + ffi::XL_BUFFER_TOO_SMALL => { let needed = if written > 0 { written as usize } else { self.buffer.len() * 2 }; if needed <= self.buffer.len() { - return Some(Err(Error::from_status( - XL_ERROR, - "native asked for a buffer no larger than the current one".to_string(), + return Some(Err(Error::new( + ErrorKind::Other, + "native asked for a buffer no larger than the current one", ))); } self.buffer.resize(needed, 0); @@ -288,74 +287,19 @@ impl<'w> RowCursor<'w> { impl Drop for RowCursor<'_> { fn drop(&mut self) { - unsafe { crate::xl_rows_close(self.rows) }; - } -} - -/// Every row of a sheet, decoded natively in one call. -/// -/// Owns the native allocation and releases it on drop. Rows and cells borrow from it, so they -/// cannot outlive it. -pub struct DecodedRows { - raw: crate::XlRowsDecoded, -} - -impl DecodedRows { - pub(crate) fn new(raw: crate::XlRowsDecoded) -> DecodedRows { - DecodedRows { raw } - } - - #[must_use] - pub fn len(&self) -> usize { - if self.raw.row_count > 0 { self.raw.row_count as usize } else { 0 } - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[must_use] - pub fn get(&self, index: usize) -> Option> { - if index >= self.len() || self.raw.rows.is_null() { - return None; - } - let row = unsafe { &*self.raw.rows.add(index) }; - Some(unsafe { RowRef::from_decoded(row.cells, row.cell_count) }) - } - - pub fn iter(&self) -> impl Iterator> + '_ { - (0..self.len()).filter_map(move |index| self.get(index)) - } -} - -impl Drop for DecodedRows { - fn drop(&mut self) { - unsafe { crate::xl_free_rows(&mut self.raw) }; - } -} - -// SAFETY: plain native allocations, read only through &self, released by xl_free_rows, which any -unsafe impl Send for DecodedRows {} -unsafe impl Sync for DecodedRows {} - -impl std::fmt::Debug for DecodedRows { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DecodedRows").field("len", &self.len()).finish() + unsafe { ffi::xl_rows_close(self.rows) }; } } /// Rows are usually well under this; it only sets how often an oversized sheet costs a retry. const INITIAL_ALL_ROWS_BUFFER: usize = 1024 * 1024; -/// Every row of a sheet, read into one flat buffer by a single `xl_rows_read_all_blob` call. +/// Every row of a sheet, from [`Sheet::read_all`](crate::Sheet::read_all): one native call into +/// one flat buffer this type owns as a plain `Vec`. /// -/// Unlike [`DecodedRows`], the native side allocates nothing per row/cell here - `xl_rows_read_all_blob` -/// writes the same wire format `RowCursor::next_row` decodes, one row after another with a length -/// prefix, into a buffer this type owns as a plain `Vec`. That means no `xl_free_*` call on -/// drop (there is nothing native to release) and one native allocation total instead of one per -/// row - prefer this over [`Sheet::read_all_decoded`](crate::workbook::Sheet::read_all_decoded) -/// unless something specifically needs the decoded-array shape. +/// The native side allocates nothing per row or cell - `xl_rows_read_all_blob` writes the same wire +/// format `RowCursor::next_row` decodes, one row after another with a length prefix - so there is +/// nothing native to release on drop. Rows and cells borrow from it, so they cannot outlive it. pub struct AllRows { buffer: Vec, /// Each row's byte range within `buffer`, already excluding the row's own length prefix - so @@ -364,27 +308,27 @@ pub struct AllRows { } impl AllRows { - pub(crate) fn read(rows: *mut crate::XlRows) -> Result { + pub(crate) fn read(rows: *mut ffi::XlRows) -> Result { let mut buffer = vec![0u8; INITIAL_ALL_ROWS_BUFFER]; loop { let mut written: i32 = 0; let capacity = i32::try_from(buffer.len()).unwrap_or(i32::MAX); let status = unsafe { - crate::xl_rows_read_all_blob(rows, buffer.as_mut_ptr(), capacity, &mut written) + ffi::xl_rows_read_all_blob(rows, buffer.as_mut_ptr(), capacity, &mut written) }; match status { - crate::XL_OK => { + ffi::XL_OK => { buffer.truncate(if written > 0 { written as usize } else { 0 }); let row_ranges = parse_row_ranges(&buffer)?; return Ok(AllRows { buffer, row_ranges }); } - crate::XL_BUFFER_TOO_SMALL => { + ffi::XL_BUFFER_TOO_SMALL => { let needed = if written > 0 { written as usize } else { buffer.len() * 2 }; if needed <= buffer.len() { - return Err(Error::from_status( - XL_ERROR, - "native asked for a buffer no larger than the current one".to_string(), + return Err(Error::new( + ErrorKind::Other, + "native asked for a buffer no larger than the current one", )); } buffer.resize(needed, 0); @@ -453,7 +397,7 @@ fn parse_row_ranges(buffer: &[u8]) -> Result, Error> { } fn malformed_all_rows_blob() -> Error { - Error::from_status(XL_ERROR, "native returned a malformed all-rows blob".to_string()) + Error::new(ErrorKind::Other, "native returned a malformed all-rows blob") } #[cfg(test)] diff --git a/rust/excelreader/src/source.rs b/rust/excelreader/src/source.rs index cee8f842..bd476452 100644 --- a/rust/excelreader/src/source.rs +++ b/rust/excelreader/src/source.rs @@ -1,13 +1,13 @@ //! Workbooks opened from bytes the caller serves: a positional [`Source`] or any [`Read`]. -use crate::workbook::{check, check_abi_version, Workbook}; -use crate::{Error, OpenOptions, XlSource, XlStream, XlWorkbook, XL_INVALID_ARGUMENT}; +use crate::ffi::{self, XlSource, XlStream}; +use crate::{Error, ErrorKind, OpenOptions, Workbook}; use std::any::Any; use std::ffi::c_void; use std::io::{self, Read}; use std::panic::{catch_unwind, AssertUnwindSafe}; -/// Random-access bytes for [`Workbook::open_source`]. The library calls `read_at` from several +/// Random-access bytes for [`OpenOptions::open_source`]. The library calls `read_at` from several /// threads at once, which is why the trait requires `Send + Sync`. pub trait Source: Send + Sync + 'static { /// Total size in bytes. Called once, while the workbook opens. @@ -18,47 +18,44 @@ pub trait Source: Send + Sync + 'static { fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result; } -impl Workbook { +impl OpenOptions { /// Opens a workbook over `source`, which the library owns from here on and drops once nothing /// opened from the workbook needs it. Every sheet can be read in parallel. - pub fn open_source(source: S, format: i32, options: Option<&OpenOptions>) -> Result { - check_abi_version()?; - let size = source.size().map_err(|error| Error::from_status(XL_INVALID_ARGUMENT, error.to_string()))?; - let length = i64::try_from(size) - .map_err(|_| Error::from_status(XL_INVALID_ARGUMENT, "the source size does not fit in an i64".to_string()))?; - let raw = XlSource { - struct_size: std::mem::size_of::() as i32, - user_data: Box::into_raw(Box::new(source)).cast(), - length, - read_at: Some(read_at::), - release: Some(release::), - }; - let options = options.map(OpenOptions::to_raw); - let options_ptr = options - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - check(unsafe { crate::xl_open_source(&raw, format, options_ptr, &mut handle) })?; - Ok(Workbook::from_handle(handle)) + pub fn open_source(&self, source: S) -> Result { + let size = source + .size() + .map_err(|error| Error::new(ErrorKind::InvalidArgument, error.to_string()))?; + let length = i64::try_from(size).map_err(|_| { + Error::new( + ErrorKind::InvalidArgument, + "the source size does not fit in an i64", + ) + })?; + self.open_raw(|format, options, handle| { + let raw = XlSource { + struct_size: std::mem::size_of::() as i32, + user_data: Box::into_raw(Box::new(source)).cast(), + length, + read_at: Some(read_at::), + release: Some(release::), + }; + unsafe { ffi::xl_open_source(&raw, format, options, handle) } + }) } - /// Opens a workbook over `reader`, which cannot seek. A CSV is read as it arrives, and - /// again from the start until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole first. The library owns `reader` from here on. - pub fn open_reader(reader: R, format: i32, options: Option<&OpenOptions>) -> Result { - check_abi_version()?; - let raw = XlStream { - struct_size: std::mem::size_of::() as i32, - user_data: Box::into_raw(Box::new(reader)).cast(), - read: Some(read::), - release: Some(release::), - }; - let options = options.map(OpenOptions::to_raw); - let options_ptr = options - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - check(unsafe { crate::xl_open_stream(&raw, format, options_ptr, &mut handle) })?; - Ok(Workbook::from_handle(handle)) + /// Opens a workbook over `reader`, which cannot seek. A CSV is read as it arrives, and again + /// from the start until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole first. The + /// library owns `reader` from here on. + pub fn open_reader(&self, reader: R) -> Result { + self.open_raw(|format, options, handle| { + let raw = XlStream { + struct_size: std::mem::size_of::() as i32, + user_data: Box::into_raw(Box::new(reader)).cast(), + read: Some(read::), + release: Some(release::), + }; + unsafe { ffi::xl_open_stream(&raw, format, options, handle) } + }) } } @@ -89,6 +86,6 @@ fn report(outcome: Result, Box>) -> i64 { Ok(Err(error)) => error.to_string(), Err(_) => "source panicked".to_string(), }; - unsafe { crate::xl_set_source_error(message.as_ptr(), message.len() as i32) }; + unsafe { ffi::xl_set_source_error(message.as_ptr(), message.len() as i32) }; -1 } diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 14934735..edf7639e 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -1,14 +1,15 @@ -use crate::{ - Date, Error, OpenOptions, Time, Timestamp, XlColumn, XlColumnSpec, XlInferredSchema, XlTable, - XlWorkbook, XL_BUFFER_TOO_SMALL, XL_EOF, XL_ERROR, XL_FORMAT_AUTO, XL_OK, XL_T_BOOL, XL_T_DATE, - XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, +use crate::ffi::{ + self, XlColumn, XlColumnSpec, XlInferredSchema, XlOpenOptions, XlTable, XlWorkbook, + XL_BUFFER_TOO_SMALL, XL_EOF, XL_OK, }; +use crate::{AllRows, Date, Error, ErrorKind, OpenOptions, RowCursor, Time, Timestamp}; use std::marker::PhantomData; +use std::path::Path; pub(crate) fn last_error(code: i32) -> Error { unsafe { let mut len: i32 = 0; - let ptr = crate::xl_last_error_ptr(&mut len); + let ptr = ffi::xl_last_error_ptr(&mut len); let message = if ptr.is_null() || len <= 0 { "unknown error".to_string() } else { @@ -27,20 +28,37 @@ pub(crate) fn check(code: i32) -> Result<(), Error> { } } +/// The ABI takes every path as UTF-8 bytes. +pub(crate) fn path_str(path: &Path) -> Result<&str, Error> { + path.to_str().ok_or_else(|| { + Error::new( + ErrorKind::InvalidArgument, + format!("path {} is not valid UTF-8", path.display()), + ) + }) +} + +/// The ABI takes every length as `int32`, so an oversized input is rejected rather than truncated. +pub(crate) fn len_i32(len: usize, what: &str) -> Result { + i32::try_from(len).map_err(|_| { + Error::new( + ErrorKind::InvalidArgument, + format!("{what} is {len} bytes, past the ABI's int32 length limit"), + ) + }) +} + /// Copies a native `XlBuffer` into an owned `Vec` and releases the native allocation via -/// `xl_free_buffer` - shared by `writer::write_columns_to_memory`/`write_sheet_to_memory` and -/// `writer_handle::WriterHandle::bytes`, the two places `xl_write_typed_to_memory`/ -/// `xl_write_handle_bytes` hand back an owned buffer. `buffer.data` may be null (an empty result), -/// which `from_raw_parts` cannot take - `slice::from_raw_parts` requires a non-null, well-aligned -/// pointer even for a zero-length slice. -pub(crate) fn buffer_to_vec(mut buffer: crate::XlBuffer) -> Vec { +/// `xl_free_buffer`. `buffer.data` may be null (an empty result), which `from_raw_parts` cannot +/// take even for a zero-length slice. +pub(crate) fn buffer_to_vec(mut buffer: ffi::XlBuffer) -> Vec { let bytes = if buffer.data.is_null() || buffer.len <= 0 { Vec::new() } else { unsafe { std::slice::from_raw_parts(buffer.data, buffer.len as usize).to_vec() } }; unsafe { - crate::xl_free_buffer(&mut buffer); + ffi::xl_free_buffer(&mut buffer); } bytes } @@ -53,25 +71,25 @@ pub(crate) fn buffer_to_vec(mut buffer: crate::XlBuffer) -> Vec { /// mismatch would mean reading native memory through the wrong layout, so every constructor calls /// this first. /// -/// The result is cached: it cannot change for the lifetime of the process, and every -/// `Workbook::open` would otherwise pay an FFI call for it. +/// The result is cached: it cannot change for the lifetime of the process, and every open would +/// otherwise pay an FFI call for it. pub(crate) fn check_abi_version() -> Result<(), Error> { use std::sync::OnceLock; static CHECKED: OnceLock> = OnceLock::new(); CHECKED .get_or_init(|| { - let loaded = unsafe { crate::xl_abi_version() }; - if loaded == crate::XL_ABI_VERSION { + let loaded = unsafe { ffi::xl_abi_version() }; + if loaded == ffi::XL_ABI_VERSION { warm_up(); Ok(()) } else { - Err(Error::from_status( - XL_ERROR, + Err(Error::new( + ErrorKind::Other, format!( "ExcelReader native library reports ABI version {loaded}, but this crate \ was built against {}. Update the crate and the native library together.", - crate::XL_ABI_VERSION + ffi::XL_ABI_VERSION ), )) } @@ -97,16 +115,16 @@ fn warm_up() { let mut handle: *mut XlWorkbook = std::ptr::null_mut(); let status = unsafe { - crate::xl_open_memory( + ffi::xl_open_memory( CSV.as_ptr(), CSV.len() as i32, - crate::XL_FORMAT_CSV, + ffi::XL_FORMAT_CSV, std::ptr::null(), &mut handle, ) }; if status == XL_OK && !handle.is_null() { - unsafe { crate::xl_close(handle) }; + unsafe { ffi::xl_close(handle) }; } struct CountRows(i64); @@ -121,11 +139,13 @@ fn warm_up() { } } - let options = crate::CsvParallelOptions { - degree_of_parallelism: 2, - ..crate::CsvParallelOptions::default() - }; - let _ = crate::aggregate::aggregate_csv_memory_unchecked(CSV, || CountRows(0), &options); + let _ = crate::aggregate::aggregate_unchecked( + crate::CsvInput::Bytes(CSV), + &OpenOptions::new(), + 0, + 2, + || CountRows(0), + ); } /// An open workbook. Safe to share between threads: every read goes through a [`Sheet`] and opens @@ -134,112 +154,36 @@ pub struct Workbook { handle: *mut XlWorkbook, } -// SAFETY: ABI v6 documents the workbook handle as usable from several threads at once. The only +// SAFETY: ABI v6 documents the workbook handle as usable from several threads at once. unsafe impl Send for Workbook {} unsafe impl Sync for Workbook {} impl Workbook { - pub(crate) fn from_handle(handle: *mut XlWorkbook) -> Workbook { - Workbook { handle } - } - - /// Opens `path`, sniffing the format and using every library default. - /// - /// Sniffing does NOT detect CSV - open one with [`open_with`](Self::open_with) and - /// [`XL_FORMAT_CSV`](crate::XL_FORMAT_CSV). - pub fn open(path: &str) -> Result { - Self::open_with(path, XL_FORMAT_AUTO, None) - } - - /// Opens `path` with an explicit format and optional [`OpenOptions`]. - pub fn open_with( - path: &str, - format: i32, - options: Option<&OpenOptions>, - ) -> Result { - check_abi_version()?; - let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - let status = unsafe { - crate::xl_open_file( - path.as_ptr(), - path.len() as i32, - format, - raw_ptr, - &mut handle, - ) - }; - check(status)?; - Ok(Workbook { handle }) - } - - /// In-memory equivalent of [`open_with`](Self::open_with): `data` is copied by the native - /// library, so it need not outlive this call. - pub fn open_memory( - data: &[u8], - format: i32, - options: Option<&OpenOptions>, - ) -> Result { - check_abi_version()?; - let raw = options.map(OpenOptions::to_raw); - let raw_ptr = raw - .as_ref() - .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); - let mut handle: *mut XlWorkbook = std::ptr::null_mut(); - let status = unsafe { - crate::xl_open_memory( - data.as_ptr(), - data.len() as i32, - format, - raw_ptr, - &mut handle, - ) - }; - check(status)?; - Ok(Workbook { handle }) - } - - /// Number of sheets in the workbook. - pub fn sheet_count(&self) -> Result { - let mut count: i32 = 0; - check(unsafe { crate::xl_sheet_count(self.handle, &mut count) })?; - Ok(count) - } - - /// Name of the sheet at `index`. - pub fn sheet_name_at(&self, index: i32) -> Result { - self.fill_string(|handle, buffer, capacity, out_len| unsafe { - crate::xl_sheet_name_at(handle, index, buffer, capacity, out_len) - }) - } - - /// Every sheet name, in workbook order. - pub fn sheet_names(&self) -> Result, Error> { - (0..self.sheet_count()?) - .map(|index| self.sheet_name_at(index)) - .collect() + /// Opens `path`, sniffing the format and using every library default. Sniffing does not detect + /// CSV: open one with [`OpenOptions::format`] set to [`Format::Csv`](crate::Format::Csv). + pub fn open(path: impl AsRef) -> Result { + OpenOptions::new().open(path) } /// Whether the workbook uses the 1904 date system - needed to interpret raw Excel serial dates. pub fn is_date1904(&self) -> Result { let mut flag: i32 = 0; - check(unsafe { crate::xl_is_date1904(self.handle, &mut flag) })?; + check(unsafe { ffi::xl_is_date1904(self.handle, &mut flag) })?; Ok(flag != 0) } /// The sheet at `index`. An index outside the workbook is an error here, not at the first read. pub fn sheet(&self, index: i32) -> Result, Error> { let mut visibility: i32 = 0; - check(unsafe { crate::xl_sheet_visibility_at(self.handle, index, &mut visibility) })?; + check(unsafe { ffi::xl_sheet_visibility_at(self.handle, index, &mut visibility) })?; Ok(Sheet { workbook: self, index }) } /// Every sheet, in workbook order. Opens nothing. pub fn sheets(&self) -> Result>, Error> { - Ok((0..self.sheet_count()?) + let mut count: i32 = 0; + check(unsafe { ffi::xl_sheet_count(self.handle, &mut count) })?; + Ok((0..count) .map(|index| Sheet { workbook: self, index }) .collect()) } @@ -248,7 +192,7 @@ impl Workbook { pub fn sheet_by_name(&self, name: &str) -> Result>, Error> { let mut index: i32 = -1; check(unsafe { - crate::xl_sheet_index(self.handle, name.as_ptr(), name.len() as i32, &mut index) + ffi::xl_sheet_index(self.handle, name.as_ptr(), name.len() as i32, &mut index) })?; Ok((index >= 0).then_some(Sheet { workbook: self, index })) } @@ -272,12 +216,7 @@ impl Workbook { let buffer_slice = &buffer[..len.max(0) as usize]; return str::from_utf8(buffer_slice) .map(|s| s.to_string()) - .map_err(|e| { - Error::from_status( - XL_ERROR, - format!("native library returned a non-UTF-8 name: {e}"), - ) - }); + .map_err(|e| non_utf8_name(&e)); } let mut vec_buffer = vec![0u8; len.max(0) as usize]; status = call( @@ -288,13 +227,48 @@ impl Workbook { ); check(status)?; vec_buffer.truncate(len.max(0) as usize); - String::from_utf8(vec_buffer).map_err(|e| { - Error::from_status( - XL_ERROR, - format!("native library returned a non-UTF-8 name: {e}"), - ) + String::from_utf8(vec_buffer).map_err(|e| non_utf8_name(&e)) + } +} + +fn non_utf8_name(error: &dyn std::fmt::Display) -> Error { + Error::new( + ErrorKind::Other, + format!("native library returned a non-UTF-8 name: {error}"), + ) +} + +impl OpenOptions { + /// Opens the workbook at `path` with these options. + pub fn open(&self, path: impl AsRef) -> Result { + let path = path_str(path.as_ref())?; + let path_len = len_i32(path.len(), "path")?; + self.open_raw(|format, options, handle| unsafe { + ffi::xl_open_file(path.as_ptr(), path_len, format, options, handle) + }) + } + + /// Opens a workbook held in memory. The native library copies `data`, so it need not outlive + /// this call. + pub fn open_bytes(&self, data: &[u8]) -> Result { + let data_len = len_i32(data.len(), "buffer")?; + self.open_raw(|format, options, handle| unsafe { + ffi::xl_open_memory(data.as_ptr(), data_len, format, options, handle) }) } + + /// The tail every open shares: the ABI check, lowering these options, and turning the status + /// and handle into a `Workbook`. `open` runs only after the ABI check passes. + pub(crate) fn open_raw( + &self, + open: impl FnOnce(i32, *const XlOpenOptions, *mut *mut XlWorkbook) -> i32, + ) -> Result { + check_abi_version()?; + let raw = self.to_raw(); + let mut handle: *mut XlWorkbook = std::ptr::null_mut(); + check(open(self.format_raw(), raw.as_ptr(), &mut handle))?; + Ok(Workbook { handle }) + } } /// How a sheet is shown in the application's tab bar. @@ -322,7 +296,10 @@ impl<'w> Sheet<'w> { /// The sheet's name. pub fn name(&self) -> Result { - self.workbook.sheet_name_at(self.index) + self.workbook + .fill_string(|handle, buffer, capacity, out_len| unsafe { + ffi::xl_sheet_name_at(handle, self.index, buffer, capacity, out_len) + }) } /// Whether the sheet is visible, hidden or very hidden. An error if the native side reports a @@ -330,14 +307,14 @@ impl<'w> Sheet<'w> { pub fn visibility(&self) -> Result { let mut raw: i32 = 0; check(unsafe { - crate::xl_sheet_visibility_at(self.workbook.handle, self.index, &mut raw) + ffi::xl_sheet_visibility_at(self.workbook.handle, self.index, &mut raw) })?; match raw { - crate::XL_SHEET_VISIBLE => Ok(SheetVisibility::Visible), - crate::XL_SHEET_HIDDEN => Ok(SheetVisibility::Hidden), - crate::XL_SHEET_VERY_HIDDEN => Ok(SheetVisibility::VeryHidden), - other => Err(Error::from_status( - XL_ERROR, + ffi::XL_SHEET_VISIBLE => Ok(SheetVisibility::Visible), + ffi::XL_SHEET_HIDDEN => Ok(SheetVisibility::Hidden), + ffi::XL_SHEET_VERY_HIDDEN => Ok(SheetVisibility::VeryHidden), + other => Err(Error::new( + ErrorKind::Other, format!("native library returned an unknown sheet visibility {other}"), )), } @@ -345,37 +322,55 @@ impl<'w> Sheet<'w> { /// A row-at-a-time reader over this sheet, from its first row. Each call opens an independent /// cursor, so several may be alive at once, on this thread or on others. - pub fn rows(&self) -> Result, Error> { - crate::rows::RowCursor::open(self.workbook.handle, self.index) + pub fn rows(&self) -> Result, Error> { + RowCursor::open(self.workbook.handle, self.index) } - /// Every row of this sheet in one native call, avoiding a round-trip per row. - pub fn read_all_decoded(&self) -> Result { + /// Every row of this sheet, read in one native call into a buffer the result owns outright. + pub fn read_all(&self) -> Result { let cursor = self.rows()?; - let mut raw = crate::XlRowsDecoded { row_count: 0, rows: std::ptr::null_mut() }; - check(unsafe { crate::xl_rows_read_all_decoded(cursor.raw(), &mut raw) })?; - Ok(crate::rows::DecodedRows::new(raw)) + AllRows::read(cursor.raw()) } - /// Every row of this sheet, read in one native call into a buffer this type owns outright. - /// Prefer this over [`read_all_decoded`](Self::read_all_decoded) unless something specifically - /// needs the decoded-array shape. - pub fn read_all_blob(&self) -> Result { - let cursor = self.rows()?; - crate::rows::AllRows::read(cursor.raw()) + /// Schema-driven columnar parse of this sheet into `T`s, matching C++'s `xl::parse_sheet`. + /// The fastest way to read a sheet whose columns you know: it converts on the native side and + /// never formats a cell to text. + /// + /// `header_row` is 1-based, `0` meaning no header (columns are then bound by position). + /// `degree_of_parallelism` is `0` for every processor, `1` sequential, `n` up to `n` threads, + /// negative an error; the table is the same whatever it is. + pub fn parse( + &self, + header_row: i32, + degree_of_parallelism: i32, + ) -> Result, Error> { + let arena = build_specs::(); + let mut table = empty_table(); + check(unsafe { + ffi::xl_parse_typed( + self.workbook.handle, + self.index, + arena.specs.as_ptr(), + arena.specs.len() as i32, + header_row, + degree_of_parallelism, + &mut table, + ) + })?; + TableView::new(table, arena.bindings, "xl_parse_typed") } - /// [`parse_sheet`] delivered a batch at a time. `batch_size` is rows per batch: 0 unbounded, - /// negative an error. The header row is consumed once, here, not re-read per batch. + /// [`parse`](Self::parse) delivered a batch at a time. `batch_size` is rows per batch: 0 + /// unbounded, negative an error. The header row is consumed once, here, not re-read per batch. pub fn typed_chunks( &self, header_row: i32, batch_size: i64, ) -> Result, Error> { let arena = build_specs::(); - let mut reader: *mut crate::XlTypedReader = std::ptr::null_mut(); + let mut reader: *mut ffi::XlTypedReader = std::ptr::null_mut(); check(unsafe { - crate::xl_typed_reader_open( + ffi::xl_typed_reader_open( self.workbook.handle, self.index, arena.specs.as_ptr(), @@ -393,42 +388,26 @@ impl<'w> Sheet<'w> { }) } - /// Guesses a [`parse_sheet`] schema by sampling this sheet. + /// Guesses a [`parse`](Self::parse) schema by sampling this sheet. /// - /// `header_row` has the same meaning as in [`parse_sheet`] (0 = no header); `sample_size` - /// bounds how many rows after the header are inspected. This is a guess over a sample, not a - /// guarantee - always check it fits before trusting it against the full sheet. + /// `header_row` has the same meaning as in `parse`; `sample_size` bounds how many rows after + /// the header are inspected. With `parse_text`, cells that hold text are typed too - every CSV + /// field, or numbers stored as text: integers, decimals, `true`/`false` and ISO dates or + /// date-times, when the text has exactly that shape (leading-zero codes such as `00123` stay + /// text). This is a guess over a sample, not a guarantee. pub fn infer_schema( &self, header_row: i32, sample_size: i32, + parse_text: bool, ) -> Result, Error> { - self.infer_schema_flags(header_row, sample_size, 0) - } - - /// [`infer_schema`](Self::infer_schema), also typing cells that hold text — every CSV field, - /// or numbers stored as text: integers, decimals, `true`/`false` and ISO dates or date-times, - /// when the text has exactly that shape. Leading-zero codes such as `00123` stay text. - pub fn infer_schema_parse_text( - &self, - header_row: i32, - sample_size: i32, - ) -> Result, Error> { - self.infer_schema_flags(header_row, sample_size, crate::XL_INFER_PARSE_TEXT) - } - - fn infer_schema_flags( - &self, - header_row: i32, - sample_size: i32, - flags: i32, - ) -> Result, Error> { + let flags = if parse_text { ffi::XL_INFER_PARSE_TEXT } else { 0 }; let mut schema = XlInferredSchema { columns: std::ptr::null_mut(), column_count: 0, }; check(unsafe { - crate::xl_infer_schema( + ffi::xl_infer_schema( self.workbook.handle, self.index, header_row, @@ -439,10 +418,11 @@ impl<'w> Sheet<'w> { })?; let columns = unsafe { copy_inferred(&schema) }; - unsafe { crate::xl_free_schema(&mut schema) }; - Ok(columns) + unsafe { ffi::xl_free_schema(&mut schema) }; + columns } + #[cfg(feature = "arrow")] pub(crate) fn handle(&self) -> *mut XlWorkbook { self.workbook.handle } @@ -455,20 +435,71 @@ impl std::fmt::Debug for Sheet<'_> { } } +/// A column's value type, shared by reading ([`InferredColumn`]) and writing ([`ColumnData`]). +/// +/// [`ColumnData`]: crate::ColumnData +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ColumnType { + String, + I64, + F64, + Bool, + /// Days since 1970-01-01. + Date, + /// Microseconds since midnight. + Time, + /// Microseconds since 1970-01-01T00:00:00Z. + Timestamp, +} + +impl ColumnType { + pub(crate) fn raw(self) -> i32 { + match self { + ColumnType::String => ffi::XL_T_STRING, + ColumnType::I64 => ffi::XL_T_I64, + ColumnType::F64 => ffi::XL_T_F64, + ColumnType::Bool => ffi::XL_T_BOOL, + ColumnType::Date => ffi::XL_T_DATE, + ColumnType::Time => ffi::XL_T_TIME, + ColumnType::Timestamp => ffi::XL_T_TIMESTAMP, + } + } + + fn from_raw(value: i32) -> Option { + match value { + ffi::XL_T_STRING => Some(ColumnType::String), + ffi::XL_T_I64 => Some(ColumnType::I64), + ffi::XL_T_F64 => Some(ColumnType::F64), + ffi::XL_T_BOOL => Some(ColumnType::Bool), + ffi::XL_T_DATE => Some(ColumnType::Date), + ffi::XL_T_TIME => Some(ColumnType::Time), + ffi::XL_T_TIMESTAMP => Some(ColumnType::Timestamp), + _ => None, + } + } +} + /// Deep-copies a native-owned inferred schema into owned Rust values, so the caller can free the /// native allocation immediately. /// /// # Safety /// `schema` must be one `xl_infer_schema` returned `XL_OK` for, not yet passed to `xl_free_schema`. -unsafe fn copy_inferred(schema: &XlInferredSchema) -> Vec { +unsafe fn copy_inferred(schema: &XlInferredSchema) -> Result, Error> { if schema.columns.is_null() || schema.column_count <= 0 { - return Vec::new(); + return Ok(Vec::new()); } let specs = std::slice::from_raw_parts(schema.columns, schema.column_count as usize); specs .iter() - .map(|spec| InferredColumn { - name: if spec.name_count <= 0 || spec.names.is_null() || spec.name_lens.is_null() { + .map(|spec| { + let column_type = ColumnType::from_raw(spec.r#type).ok_or_else(|| { + Error::new( + ErrorKind::Other, + format!("native library inferred an unknown column type {}", spec.r#type), + ) + })?; + let name = if spec.name_count <= 0 || spec.names.is_null() || spec.name_lens.is_null() { None } else { let name_ptr = *spec.names; @@ -479,10 +510,13 @@ unsafe fn copy_inferred(schema: &XlInferredSchema) -> Vec { let bytes = std::slice::from_raw_parts(name_ptr, name_len as usize); Some(String::from_utf8_lossy(bytes).into_owned()) } - }, - index: spec.index, - column_type: spec.r#type, - nullable: spec.nullable != 0, + }; + Ok(InferredColumn { + name, + index: spec.index, + column_type, + nullable: spec.nullable != 0, + }) }) .collect() } @@ -493,8 +527,7 @@ pub struct InferredColumn { /// Header text, or `None` when the column must be resolved by [`index`](Self::index) instead. pub name: Option, pub index: i32, - /// One of the `XL_T_*` constants. - pub column_type: i32, + pub column_type: ColumnType, pub nullable: bool, } @@ -502,7 +535,7 @@ impl Drop for Workbook { fn drop(&mut self) { if !self.handle.is_null() { unsafe { - crate::xl_close(self.handle); + ffi::xl_close(self.handle); } self.handle = std::ptr::null_mut(); } @@ -519,11 +552,11 @@ impl std::fmt::Debug for Workbook { } } -/// One field <-> column binding for `T`. Construct via `ExcelMapper::bindings()`. +/// One field <-> column binding for `T`, produced by `#[derive(ExcelMapper)]`. pub struct ColumnBinding { /// Candidate header names, in priority order — the first one present in the header row wins. pub names: &'static [&'static str], - pub xl_type: i32, + pub column_type: ColumnType, pub assign: fn(&mut T, &XlColumn, i64), } @@ -536,10 +569,10 @@ impl Clone for ColumnBinding { impl Copy for ColumnBinding {} -/// Implemented by any struct `parse_sheet` can populate from a `parse_typed` result. Implement -/// `bindings()` by hand (mirroring `xl::ExcelMapper` on the C++ side), or derive it with -/// `#[derive(ExcelMapper)]` (see `excelreader_derive`). +/// Implemented by any struct [`Sheet::parse`] can populate. Derive it with +/// `#[derive(ExcelMapper)]`, which also implements [`ExcelWriter`](crate::ExcelWriter). pub trait ExcelMapper: Sized { + #[doc(hidden)] fn bindings() -> Vec>; } @@ -559,8 +592,8 @@ fn is_valid(col: &XlColumn, row: i64) -> bool { /// /// These are safe `pub` functions that index into raw native buffers, so this check is what makes /// them sound: without it a `row` past the column returns whatever sits after the allocation. The -/// type half catches the other class of mistake - reading an `XL_T_F64` buffer as `i64` is an -/// `ExcelMapper::bindings()` bug, not recoverable input. +/// type half catches the other class of mistake - reading an `XL_T_F64` buffer as `i64` is a +/// bindings bug, not recoverable input. #[inline] fn check_access(col: &XlColumn, row: i64, expected_type: i32, accessor: &str) { assert_eq!( @@ -580,13 +613,12 @@ fn check_access(col: &XlColumn, row: i64, expected_type: i32, accessor: &str) { /// The ABI documents every string it returns as UTF-8, but this validates rather than trusting it: /// the bytes cross an FFI boundary from a binary resolved at build time, and `from_utf8_unchecked` /// on a value that turns out not to be UTF-8 is undefined behavior, not merely a wrong answer. -/// Validation is a linear scan over bytes already in cache. /// /// # Panics -/// If `col` is not an `XL_T_STRING` column, if `row` is out of bounds, or if the native library -/// returned bytes that are not valid UTF-8 (an ABI contract violation). +/// If `col` is not a string column, if `row` is out of bounds, or if the native library returned +/// bytes that are not valid UTF-8 (an ABI contract violation). pub fn column_str(col: &XlColumn, row: i64) -> &str { - check_access(col, row, XL_T_STRING, "column_str"); + check_access(col, row, ffi::XL_T_STRING, "column_str"); unsafe { let offsets = col.values as *const i32; let start = *offsets.offset(row as isize); @@ -599,50 +631,58 @@ pub fn column_str(col: &XlColumn, row: i64) -> &str { } /// # Panics -/// If `col` is not an `XL_T_I64` column, or `row` is out of bounds. +/// If `col` is not an `i64` column, or `row` is out of bounds. pub fn column_i64(col: &XlColumn, row: i64) -> i64 { - check_access(col, row, XL_T_I64, "column_i64"); + check_access(col, row, ffi::XL_T_I64, "column_i64"); unsafe { *(col.values as *const i64).offset(row as isize) } } /// # Panics -/// If `col` is not an `XL_T_F64` column, or `row` is out of bounds. +/// If `col` is not an `f64` column, or `row` is out of bounds. pub fn column_f64(col: &XlColumn, row: i64) -> f64 { - check_access(col, row, XL_T_F64, "column_f64"); + check_access(col, row, ffi::XL_T_F64, "column_f64"); unsafe { *(col.values as *const f64).offset(row as isize) } } /// # Panics -/// If `col` is not an `XL_T_BOOL` column, or `row` is out of bounds. +/// If `col` is not a bool column, or `row` is out of bounds. pub fn column_bool(col: &XlColumn, row: i64) -> bool { - check_access(col, row, XL_T_BOOL, "column_bool"); + check_access(col, row, ffi::XL_T_BOOL, "column_bool"); unsafe { *(col.values as *const u8).offset(row as isize) != 0 } } /// # Panics -/// If `col` is not an `XL_T_DATE` column, or `row` is out of bounds. +/// If `col` is not a date column, or `row` is out of bounds. pub fn column_date(col: &XlColumn, row: i64) -> Date { - check_access(col, row, XL_T_DATE, "column_date"); + check_access(col, row, ffi::XL_T_DATE, "column_date"); Date::new(unsafe { *(col.values as *const i32).offset(row as isize) }) } /// # Panics -/// If `col` is not an `XL_T_TIME` column, or `row` is out of bounds. +/// If `col` is not a time column, or `row` is out of bounds. pub fn column_time(col: &XlColumn, row: i64) -> Time { - check_access(col, row, XL_T_TIME, "column_time"); + check_access(col, row, ffi::XL_T_TIME, "column_time"); Time::new(unsafe { *(col.values as *const i64).offset(row as isize) }) } /// # Panics -/// If `col` is not an `XL_T_TIMESTAMP` column, or `row` is out of bounds. +/// If `col` is not a timestamp column, or `row` is out of bounds. pub fn column_timestamp(col: &XlColumn, row: i64) -> Timestamp { - check_access(col, row, XL_T_TIMESTAMP, "column_timestamp"); + check_access(col, row, ffi::XL_T_TIMESTAMP, "column_timestamp"); Timestamp::new(unsafe { *(col.values as *const i64).offset(row as isize) }) } -/// Owns a `parse_typed` result. Frees the native table on `Drop`. `iter()` builds one `T` per call -/// to `next()` from the columnar buffers - no upfront `Vec` allocation beyond what -/// `xl_parse_typed` itself already made. +fn empty_table() -> XlTable { + XlTable { + column_count: 0, + row_count: 0, + columns: std::ptr::null_mut(), + } +} + +/// Owns a typed parse result. Frees the native table on `Drop`. `iter()` builds one `T` per call +/// to `next()` from the columnar buffers - no upfront `Vec` allocation beyond what the native +/// parse itself already made. pub struct TableView { table: XlTable, bindings: Vec>, @@ -650,6 +690,31 @@ pub struct TableView { } impl TableView { + /// Takes ownership of `table`, freeing it and failing if its column count disagrees with the + /// bindings it will be read through. + fn new( + mut table: XlTable, + bindings: Vec>, + source: &str, + ) -> Result { + if table.column_count as usize != bindings.len() { + let column_count = table.column_count; + unsafe { ffi::xl_free_table(&mut table) }; + return Err(Error::new( + ErrorKind::Other, + format!( + "{source} returned {column_count} columns for {} specs", + bindings.len() + ), + )); + } + Ok(TableView { + table, + bindings, + _marker: PhantomData, + }) + } + pub fn len(&self) -> i64 { self.table.row_count } @@ -692,12 +757,13 @@ impl TableView { impl Drop for TableView { fn drop(&mut self) { unsafe { - crate::xl_free_table(&mut self.table); + ffi::xl_free_table(&mut self.table); } } } // SAFETY: plain native allocations, read only through &self, released by xl_free_table, which any +// thread may call. unsafe impl Send for TableView {} unsafe impl Sync for TableView {} @@ -714,13 +780,14 @@ impl std::fmt::Debug for TableView { /// Iterator over the batches of one chunked typed read. Closes the native reader on `Drop`. pub struct TypedChunks<'a, T: ExcelMapper> { - reader: *mut crate::XlTypedReader, + reader: *mut ffi::XlTypedReader, bindings: Vec>, done: bool, _workbook: PhantomData<&'a Workbook>, } // SAFETY: a typed reader owns its native enumerator; ABI v6 lets it be used from any one thread +// at a time. unsafe impl Send for TypedChunks<'_, T> {} impl Iterator for TypedChunks<'_, T> { @@ -731,12 +798,8 @@ impl Iterator for TypedChunks<'_, T> { return None; } - let mut table = XlTable { - column_count: 0, - row_count: 0, - columns: std::ptr::null_mut(), - }; - let status = unsafe { crate::xl_typed_reader_next(self.reader, &mut table) }; + let mut table = empty_table(); + let status = unsafe { ffi::xl_typed_reader_next(self.reader, &mut table) }; if status == XL_EOF { self.done = true; return None; @@ -746,31 +809,16 @@ impl Iterator for TypedChunks<'_, T> { return Some(Err(last_error(status))); } - if table.column_count as usize != self.bindings.len() { - let column_count = table.column_count; - unsafe { crate::xl_free_table(&mut table) }; - self.done = true; - return Some(Err(Error::from_status( - XL_ERROR, - format!( - "xl_typed_reader_next returned {column_count} columns for {} specs", - self.bindings.len() - ), - ))); - } - - // ponytail: one small Vec of function pointers cloned per batch, not per row. Give - Some(Ok(TableView { - table, - bindings: self.bindings.clone(), - _marker: PhantomData, - })) + // ponytail: one small Vec of function pointers cloned per batch, not per row. + let view = TableView::new(table, self.bindings.clone(), "xl_typed_reader_next"); + self.done = view.is_err(); + Some(view) } } impl Drop for TypedChunks<'_, T> { fn drop(&mut self) { - unsafe { crate::xl_typed_reader_close(self.reader) }; + unsafe { ffi::xl_typed_reader_close(self.reader) }; } } @@ -840,7 +888,7 @@ pub(crate) fn build_specs() -> SpecArena { name_lens: name_lens[i].as_ptr(), name_count: b.names.len() as i32, index: 0, - r#type: b.xl_type, + r#type: b.column_type.raw(), nullable: 1, }) .collect(); @@ -851,60 +899,3 @@ pub(crate) fn build_specs() -> SpecArena { _name_lens: name_lens, } } - -/// Schema-driven columnar parse of one sheet, matching C++'s `xl::parse_sheet`. -pub fn parse_sheet( - sheet: Sheet<'_>, - header_row: i32, -) -> Result, Error> { - parse_sheet_parallel(sheet, header_row, 1) -} - -/// [`parse_sheet`] with a degree of parallelism: `0` uses every processor, `1` is sequential, `n` -/// up to `n` threads, negative is an error. Only CSV is split; other formats read sequentially. -/// The table equals `parse_sheet`'s. -pub fn parse_sheet_parallel( - sheet: Sheet<'_>, - header_row: i32, - degree_of_parallelism: i32, -) -> Result, Error> { - let arena = build_specs::(); - let bindings = arena.bindings; - let mut table = XlTable { - column_count: 0, - row_count: 0, - columns: std::ptr::null_mut(), - }; - unsafe { - let status = crate::xl_parse_typed( - sheet.handle(), - sheet.index(), - arena.specs.as_ptr(), - arena.specs.len() as i32, - header_row, - degree_of_parallelism, - &mut table, - ); - if status != XL_OK { - return Err(last_error(status)); - } - } - - if table.column_count as usize != bindings.len() { - let column_count = table.column_count; - unsafe { crate::xl_free_table(&mut table) }; - return Err(Error::from_status( - XL_ERROR, - format!( - "xl_parse_typed returned {column_count} columns for {} specs", - bindings.len() - ), - )); - } - - Ok(TableView { - table, - bindings, - _marker: PhantomData, - }) -} diff --git a/rust/excelreader/src/writer.rs b/rust/excelreader/src/writer.rs index 69a95618..da975dd5 100644 --- a/rust/excelreader/src/writer.rs +++ b/rust/excelreader/src/writer.rs @@ -1,4 +1,4 @@ -//! Writing a columnar table to a workbook file, through the ABI's single `xl_write_typed` export. +//! Writing a columnar table to a workbook, through the ABI's single-shot `xl_write_typed` exports. //! //! Two layers sit here. [`write_columns`] takes buffers the caller already owns and hands them //! straight to the ABI - the borrow is the whole point, and the lifetimes on [`Column`] are what @@ -6,13 +6,12 @@ //! something the compiler enforces. [`write_sheet`] sits on top for callers holding a slice of //! structs, and pays exactly one transpose to get them into columns. -use crate::workbook::{check, check_abi_version}; -use crate::{ - Error, WriteOptions, XlColumn, XlColumnSpec, XlTable, XL_FORMAT_AUTO, XL_FORMAT_CSV, - XL_FORMAT_XLS, XL_FORMAT_XLSB, XL_FORMAT_XLSX, XL_INVALID_ARGUMENT, XL_T_BOOL, XL_T_DATE, - XL_T_F64, XL_T_I64, XL_T_STRING, XL_T_TIME, XL_T_TIMESTAMP, -}; +use crate::ffi::{self, XlColumn, XlColumnSpec, XlTable}; +use crate::workbook::{buffer_to_vec, check, check_abi_version, len_i32, path_str}; +use crate::{ColumnType, Error, ErrorKind, WriteOptions}; +use std::io::Write; use std::os::raw::c_void; +use std::path::Path; /// One column's buffers, borrowed from the caller. Each variant's slice is that column type's /// exact wire layout (see `excelreader.h`), so nothing is converted on the way out. @@ -56,17 +55,17 @@ impl ColumnData<'_> { } /// The `XL_T_*` tag this variant writes as. - #[must_use] - pub fn xl_type(&self) -> i32 { + pub(crate) fn xl_type(&self) -> i32 { match self { - ColumnData::Str { .. } => XL_T_STRING, - ColumnData::I64(_) => XL_T_I64, - ColumnData::F64(_) => XL_T_F64, - ColumnData::Bool(_) => XL_T_BOOL, - ColumnData::Date(_) => XL_T_DATE, - ColumnData::Time(_) => XL_T_TIME, - ColumnData::Timestamp(_) => XL_T_TIMESTAMP, + ColumnData::Str { .. } => ColumnType::String, + ColumnData::I64(_) => ColumnType::I64, + ColumnData::F64(_) => ColumnType::F64, + ColumnData::Bool(_) => ColumnType::Bool, + ColumnData::Date(_) => ColumnType::Date, + ColumnData::Time(_) => ColumnType::Time, + ColumnData::Timestamp(_) => ColumnType::Timestamp, } + .raw() } fn pointers(&self) -> (*const c_void, *const u8, i64) { @@ -110,27 +109,8 @@ impl Column<'_> { } } -/// Infers an `XL_FORMAT_*` from a path's extension, case-insensitively. Returns -/// [`XL_FORMAT_AUTO`] when the extension is absent or unrecognized - and since `xl_write_typed` -/// rejects `AUTO` with a message of its own, an unrecognized path fails the write rather than -/// silently picking a format. -#[must_use] -pub fn format_from_path(path: &str) -> i32 { - let name = path.rsplit(['/', '\\']).next().unwrap_or(path); - let Some((_, extension)) = name.rsplit_once('.') else { - return XL_FORMAT_AUTO; - }; - match extension.to_ascii_lowercase().as_str() { - "xlsx" => XL_FORMAT_XLSX, - "xlsb" => XL_FORMAT_XLSB, - "xls" => XL_FORMAT_XLS, - "csv" => XL_FORMAT_CSV, - _ => XL_FORMAT_AUTO, - } -} - fn invalid(message: String) -> Error { - Error::from_status(XL_INVALID_ARGUMENT, message) + Error::new(ErrorKind::InvalidArgument, message) } /// Returns the row count every column agreed on, or the first problem found. @@ -191,23 +171,18 @@ fn validate(columns: &[Column<'_>]) -> Result { Ok(rows) } -/// Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle -/// exists before or after, and every buffer reachable from `columns` and `options` is borrowed for -/// the duration of the call and never freed by the native library. -/// -/// `format` must be one of `XL_FORMAT_XLS`/`XLSX`/`XLSB`/`CSV`. [`XL_FORMAT_AUTO`] is an error: a -/// file being created has no signature bytes to sniff. Use [`format_from_path`] to infer one. -/// -/// On failure the destination may exist and be incomplete; cleaning it up is the caller's. -pub fn write_columns( - path: &str, - format: i32, - columns: &[Column<'_>], - options: Option<&WriteOptions>, -) -> Result<(), Error> { - check_abi_version()?; - let row_count = validate(columns)?; +/// The ABI's column specs and table for `columns`, borrowing every buffer. `specs` and `table` +/// point into the other three fields, so the whole value must outlive the FFI call. +struct RawTable { + specs: Vec, + table: XlTable, + _columns: Vec, + _names: Vec<*const u8>, + _name_lens: Vec, +} +fn raw_table(columns: &[Column<'_>]) -> Result { + let row_count = validate(columns)?; let names: Vec<*const u8> = columns .iter() .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) @@ -229,88 +204,89 @@ pub fn write_columns( }) .collect(); let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); - let table = XlTable { column_count: columns.len() as i32, row_count, columns: raw_columns.as_ptr().cast_mut(), }; - let raw_options = options.map(WriteOptions::to_raw); - let options_ptr = crate::options::ptr_or_null(&raw_options); + Ok(RawTable { + specs, + table, + _columns: raw_columns, + _names: names, + _name_lens: name_lens, + }) +} - let status = unsafe { - crate::xl_write_typed( +/// Writes `columns` to `path` as a single sheet, then closes the file. One-shot: no writer handle +/// exists before or after, and every buffer reachable from `columns` and `options` is borrowed for +/// the duration of the call and never freed by the native library. +/// +/// The format is [`WriteOptions::format`], or inferred from the extension when that is +/// [`Format::Auto`](crate::Format::Auto) - an unrecognized extension is then an error, since a file +/// being created has no signature bytes to sniff. +/// +/// On failure the destination may exist and be incomplete; cleaning it up is the caller's. +pub fn write_columns( + path: impl AsRef, + columns: &[Column<'_>], + options: &WriteOptions, +) -> Result<(), Error> { + check_abi_version()?; + let path = path.as_ref(); + let format = options.format_for(Some(path))?; + let path = path_str(path)?; + let path_len = len_i32(path.len(), "path")?; + let raw = raw_table(columns)?; + let raw_options = options.to_raw(); + check(unsafe { + ffi::xl_write_typed( path.as_ptr(), - path.len() as i32, + path_len, format, - specs.as_ptr(), - &table, - options_ptr, + raw.specs.as_ptr(), + &raw.table, + &raw_options, ) - }; - check(status) + }) } -/// In-memory equivalent of [`write_columns`]: same validation and column lowering, but the -/// workbook is built in memory and returned as bytes instead of being written to a path - so, -/// unlike [`write_columns`], there is no path to infer a format from and `format` is always -/// required. -/// -/// # Errors -/// Anything [`write_columns`] reports. -pub fn write_columns_to_memory( - format: i32, +/// [`write_columns`] to any [`Write`]: the workbook is built in memory, then written to `writer` +/// in one `write_all`. [`WriteOptions::format`] must be set - there is no extension to infer one +/// from. +pub fn write_columns_to( + mut writer: impl Write, columns: &[Column<'_>], - options: Option<&WriteOptions>, -) -> Result, Error> { + options: &WriteOptions, +) -> Result<(), Error> { check_abi_version()?; - let row_count = validate(columns)?; - - let names: Vec<*const u8> = columns - .iter() - .map(|c| c.name.map_or(std::ptr::null(), str::as_ptr)) - .collect(); - let name_lens: Vec = columns - .iter() - .map(|c| c.name.map_or(0, |n| n.len() as i32)) - .collect(); - let specs: Vec = columns - .iter() - .enumerate() - .map(|(index, column)| XlColumnSpec { - names: &names[index], - name_lens: &name_lens[index], - name_count: i32::from(column.name.is_some()), - index: 0, - r#type: column.data.xl_type(), - nullable: 0, - }) - .collect(); - let raw_columns: Vec = columns.iter().map(Column::to_raw).collect(); - - let table = XlTable { - column_count: columns.len() as i32, - row_count, - columns: raw_columns.as_ptr().cast_mut(), - }; - let raw_options = options.map(WriteOptions::to_raw); - let options_ptr = crate::options::ptr_or_null(&raw_options); - - let mut buffer = crate::XlBuffer { + let format = options.format_for(None)?; + let raw = raw_table(columns)?; + let raw_options = options.to_raw(); + let mut buffer = ffi::XlBuffer { data: std::ptr::null_mut(), len: 0, }; - let status = unsafe { - crate::xl_write_typed_to_memory(format, specs.as_ptr(), &table, options_ptr, &mut buffer) - }; - check(status)?; - Ok(crate::workbook::buffer_to_vec(buffer)) + check(unsafe { + ffi::xl_write_typed_to_memory( + format, + raw.specs.as_ptr(), + &raw.table, + &raw_options, + &mut buffer, + ) + })?; + let bytes = buffer_to_vec(buffer); + writer + .write_all(&bytes) + .and_then(|()| writer.flush()) + .map_err(|e| Error::io(&e)) } /// Wraps a finished plaintext XLSX/XLSB package at `package_path` in an agile-encrypted (ECMA-376 /// 4.4) CFB container, written to `destination_path` (overwriting an existing file). The result -/// opens with the same password via [`crate::OpenOptions::password`]. Encryption parameters are -/// fixed at Excel's own defaults - there are no options. +/// opens with the same password via [`OpenOptions::password`](crate::OpenOptions::password). +/// Encryption parameters are fixed at Excel's own defaults - there are no options. /// /// `package_path` is read twice, so it must already be a finished file (write it with /// [`write_columns`]/[`write_sheet`] first). @@ -318,23 +294,31 @@ pub fn write_columns_to_memory( /// # Errors /// `package_path` does not exist, is not readable, or is not a valid plaintext OOXML package; /// `password` is empty; or the usual file I/O failures on either path. -pub fn encrypt_package(package_path: &str, destination_path: &str, password: &str) -> Result<(), Error> { +pub fn encrypt_package( + package_path: impl AsRef, + destination_path: impl AsRef, + password: &str, +) -> Result<(), Error> { check_abi_version()?; - let status = unsafe { - crate::xl_encrypt_package( + let package_path = path_str(package_path.as_ref())?; + let destination_path = path_str(destination_path.as_ref())?; + let package_path_len = len_i32(package_path.len(), "package_path")?; + let destination_path_len = len_i32(destination_path.len(), "destination_path")?; + let password_len = len_i32(password.len(), "password")?; + check(unsafe { + ffi::xl_encrypt_package( package_path.as_ptr(), - package_path.len() as i32, + package_path_len, destination_path.as_ptr(), - destination_path.len() as i32, + destination_path_len, password.as_ptr(), - password.len() as i32, + password_len, ) - }; - check(status) + }) } -/// The owning twin of [`ColumnData`], produced by [`ExcelWriter::to_columns`]. A transposed range -/// of structs has to own its columns somewhere; this is that somewhere. +/// The owning twin of [`ColumnData`], produced by `#[derive(ExcelMapper)]`'s [`ExcelWriter`] impl. +/// A transposed range of structs has to own its columns somewhere; this is that somewhere. pub enum OwnedColumnData { Str { offsets: Vec, data: Vec }, I64(Vec), @@ -356,9 +340,7 @@ pub struct OwnedColumn { } impl OwnedColumn { - /// Borrows this column in the shape [`write_columns`] takes. - #[must_use] - pub fn as_column(&self) -> Column<'_> { + fn as_column(&self) -> Column<'_> { let data = match &self.data { OwnedColumnData::Str { offsets, data } => ColumnData::Str { offsets, data }, OwnedColumnData::I64(v) => ColumnData::I64(v), @@ -411,78 +393,61 @@ pub fn set_valid(validity: &mut [u8], row: usize) { validity[row / 8] |= 1 << (row % 8); } -/// Implemented by any struct [`write_sheet`] can write. Derive it with -/// `#[derive(ExcelMapper)]`, which emits this alongside the reading half, or write it by hand. +/// Implemented by any struct [`write_sheet`] can write. Derive it with `#[derive(ExcelMapper)]`, +/// which emits this alongside the reading half. pub trait ExcelWriter: Sized { /// Transposes `rows` into one column per field, in field order. /// /// # Errors /// Only for values the ABI cannot represent: a string column past 2 GiB, or an integer that /// does not fit `i64`. Ordinary data never fails here. + #[doc(hidden)] fn to_columns(rows: &[Self]) -> Result, Error>; } /// Writes `rows` to `path` as a single sheet, using the same field mapping -/// [`parse_sheet`](crate::workbook::parse_sheet) reads with. +/// [`Sheet::parse`](crate::Sheet::parse) reads with. The format is chosen as in +/// [`write_columns`]. /// /// `rows` is walked ONCE and each field appended to its own column buffer. That transpose is the /// only copy this makes - it is what the ABI's columnar shape costs a row-shaped caller. If you /// already hold columnar buffers, call [`write_columns`] and pay nothing. -/// -/// # Errors -/// Anything [`ExcelWriter::to_columns`] or [`write_columns`] reports. pub fn write_sheet( - path: &str, - format: i32, + path: impl AsRef, rows: &[T], - options: Option<&WriteOptions>, + options: &WriteOptions, ) -> Result<(), Error> { let owned = T::to_columns(rows)?; let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); - write_columns(path, format, &borrowed, options) + write_columns(path, &borrowed, options) } -/// In-memory equivalent of [`write_sheet`]: same transpose, but returns bytes instead of writing -/// to a path - see [`write_columns_to_memory`] for why `format` is always required here. -/// -/// # Errors -/// Anything [`ExcelWriter::to_columns`] or [`write_columns_to_memory`] reports. -pub fn write_sheet_to_memory( - format: i32, +/// [`write_sheet`] to any [`Write`], as [`write_columns_to`] does it. +pub fn write_sheet_to( + writer: impl Write, rows: &[T], - options: Option<&WriteOptions>, -) -> Result, Error> { + options: &WriteOptions, +) -> Result<(), Error> { let owned = T::to_columns(rows)?; let borrowed: Vec> = owned.iter().map(OwnedColumn::as_column).collect(); - write_columns_to_memory(format, &borrowed, options) + write_columns_to(writer, &borrowed, options) } #[cfg(test)] mod tests { use super::*; - #[test] - fn format_is_inferred_from_the_extension() { - assert_eq!(format_from_path("out.xlsx"), XL_FORMAT_XLSX); - assert_eq!(format_from_path("out.XLSB"), XL_FORMAT_XLSB); - assert_eq!(format_from_path("out.xls"), XL_FORMAT_XLS); - assert_eq!(format_from_path("out.csv"), XL_FORMAT_CSV); - assert_eq!(format_from_path("out.txt"), XL_FORMAT_AUTO); - assert_eq!(format_from_path("out"), XL_FORMAT_AUTO); - assert_eq!(format_from_path("v1.2/report"), XL_FORMAT_AUTO); - } - #[test] fn validate_rejects_a_short_validity_bitmap() { let values = [1i64, 2, 3, 4, 5, 6, 7, 8, 9]; - let bitmap = [0u8]; + let bitmap = [0u8]; let columns = [Column { name: Some("a"), data: ColumnData::I64(&values), validity: Some(&bitmap), }]; let error = validate(&columns).expect_err("a 1-byte bitmap cannot cover 9 rows"); - assert_eq!(error.code(), XL_INVALID_ARGUMENT); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); } #[test] @@ -531,6 +496,7 @@ mod tests { "3 rows next to a 2-row column must be rejected" ); } + #[test] fn push_str_appends_bytes_and_one_offset_per_value() { let mut offsets = vec![0i32]; diff --git a/rust/excelreader/src/writer_handle.rs b/rust/excelreader/src/writer_handle.rs index 1f8abdf7..26d9ba19 100644 --- a/rust/excelreader/src/writer_handle.rs +++ b/rust/excelreader/src/writer_handle.rs @@ -1,90 +1,79 @@ //! Row-by-row streaming write, through the ABI's `xl_writer_handle`. //! -//! [`WriterHandle`] is the streaming counterpart to [`crate::writer::write_columns`]/ -//! [`crate::writer::write_sheet`]: instead of materializing a whole table up front, it writes -//! directly as each call arrives - one sheet and one row open at a time. Call order mirrors the C -//! ABI (see `xl_writer_handle` in `excelreader.h`): `open`/`open_with`/`open_memory`, then per -//! sheet `start_sheet`..`end_sheet`, each containing `start_row`..`end_row` with one `write_*` -//! call per cell in between, left to right. A call out of order returns an `Err` rather than -//! corrupting output, and the handle stays usable afterward - fix the call order and continue, or -//! let `Drop` discard it. - -use crate::workbook::{buffer_to_vec, check, check_abi_version}; -use crate::writer::format_from_path; -use crate::{ - Date, Error, Time, Timestamp, WriteOptions, XlBuffer, XlWriterHandle, XL_T_BOOL, XL_T_DATE, - XL_T_F64, XL_T_I64, XL_T_TIME, XL_T_TIMESTAMP, -}; - -/// A streaming writer handle - the row-by-row counterpart to -/// [`Workbook`](crate::workbook::Workbook). +//! [`WriterHandle`] is the streaming counterpart to [`crate::write_columns`]/[`crate::write_sheet`]: +//! instead of materializing a whole table up front, it writes directly as each call arrives - one +//! sheet and one row open at a time. Call order mirrors the C ABI (see `xl_writer_handle` in +//! `excelreader.h`): `create`/`new`, then per sheet `start_sheet`..`end_sheet`, each containing +//! `start_row`..`end_row` with one `write_*` call per cell in between, left to right, then +//! `finish`. A call out of order returns an `Err` rather than corrupting output, and the handle +//! stays usable afterward - fix the call order and continue, or let `Drop` discard it. + +use crate::ffi::{self, XlWriterHandle}; +use crate::workbook::{buffer_to_vec, check, check_abi_version, len_i32, path_str}; +use crate::{ColumnType, Date, Error, Time, Timestamp, WriteOptions}; +use std::io::Write; +use std::path::Path; + +/// A streaming writer - the row-by-row counterpart to [`Workbook`](crate::Workbook). /// /// Not thread-safe - use one per thread, same contract as the C ABI. (The raw handle makes this /// type neither `Send` nor `Sync`, so the compiler enforces that for you.) /// -/// Dropping a `WriterHandle` closes and releases it (`xl_close_write_handle`), silently discarding -/// any error - same convention as [`Workbook`](crate::workbook::Workbook)'s `Drop`. Call -/// [`bytes`](Self::bytes) (memory-backed) or reopen the path (file-backed) to observe the actual -/// result; do not rely on the drop for that. -pub struct WriterHandle { +/// Call [`finish`](Self::finish) to complete the workbook and observe any error. Dropping a handle +/// without finishing it closes it on a best-effort basis, discarding any error, and writes nothing +/// to an `io::Write` destination. +pub struct WriterHandle<'w> { handle: *mut XlWriterHandle, + writer: Option>, } -impl WriterHandle { - /// Creates `path` (truncating it if it already exists), inferring the format from its - /// extension via [`crate::writer::format_from_path`]. - pub fn open(path: &str, options: Option<&WriteOptions>) -> Result { - Self::open_with(path, format_from_path(path), options) - } - - /// Creates `path` with an explicit format. `format` must be one of `XL_FORMAT_XLS`/`XLSX`/ - /// `XLSB`/`CSV` - [`XL_FORMAT_AUTO`](crate::XL_FORMAT_AUTO) is an error, the same as - /// [`write_columns`](crate::writer::write_columns). - pub fn open_with( - path: &str, - format: i32, - options: Option<&WriteOptions>, - ) -> Result { +impl WriterHandle<'static> { + /// Creates `path` (truncating it if it already exists). The format is + /// [`WriteOptions::format`], or inferred from the extension when that is + /// [`Format::Auto`](crate::Format::Auto). + pub fn create(path: impl AsRef, options: &WriteOptions) -> Result { check_abi_version()?; - let raw = options.map(WriteOptions::to_raw); - let raw_ptr = crate::options::ptr_or_null(&raw); + let path = path.as_ref(); + let format = options.format_for(Some(path))?; + let path = path_str(path)?; + let path_len = len_i32(path.len(), "path")?; + let raw = options.to_raw(); let mut handle: *mut XlWriterHandle = std::ptr::null_mut(); - let status = unsafe { - crate::xl_open_write_handle( - path.as_ptr(), - path.len() as i32, - format, - raw_ptr, - &mut handle, - ) - }; - check(status)?; - Ok(WriterHandle { handle }) - } - - /// In-memory equivalent of [`open_with`](Self::open_with): read the result back with - /// [`bytes`](Self::bytes). `format` is always required here - there is no path to infer one - /// from. - pub fn open_memory(format: i32, options: Option<&WriteOptions>) -> Result { + check(unsafe { + ffi::xl_open_write_handle(path.as_ptr(), path_len, format, &raw, &mut handle) + })?; + Ok(WriterHandle { + handle, + writer: None, + }) + } +} + +impl<'w> WriterHandle<'w> { + /// Builds the workbook in memory and writes it to `writer`, once, in [`finish`](Self::finish). + /// [`WriteOptions::format`] must be set - there is no extension to infer one from. + pub fn new(writer: impl Write + 'w, options: &WriteOptions) -> Result { check_abi_version()?; - let raw = options.map(WriteOptions::to_raw); - let raw_ptr = crate::options::ptr_or_null(&raw); + let format = options.format_for(None)?; + let raw = options.to_raw(); let mut handle: *mut XlWriterHandle = std::ptr::null_mut(); - let status = unsafe { crate::xl_open_write_handle_to_memory(format, raw_ptr, &mut handle) }; - check(status)?; - Ok(WriterHandle { handle }) + check(unsafe { ffi::xl_open_write_handle_to_memory(format, &raw, &mut handle) })?; + Ok(WriterHandle { + handle, + writer: Some(Box::new(writer)), + }) } /// Starts a new sheet named `name`. Must not be called again before the current sheet, if /// any, has been ended with [`end_sheet`](Self::end_sheet). pub fn start_sheet(&mut self, name: &str) -> Result<(), Error> { - check(unsafe { crate::xl_start_sheet(self.handle, name.as_ptr(), name.len() as i32) }) + check(unsafe { ffi::xl_start_sheet(self.handle, name.as_ptr(), name.len() as i32) }) } /// Starts a new row on the current sheet. Must not be called again before the current row, if /// any, has been ended with [`end_row`](Self::end_row). pub fn start_row(&mut self) -> Result<(), Error> { - check(unsafe { crate::xl_start_row(self.handle) }) + check(unsafe { ffi::xl_start_row(self.handle) }) } /// Writes the next cell of the current row as text, or a blank cell for `None`. @@ -92,14 +81,14 @@ impl WriterHandle { let (ptr, len) = value.map_or((std::ptr::null(), 0), |text| { (text.as_ptr(), text.len() as i32) }); - check(unsafe { crate::xl_write_string(self.handle, ptr, len) }) + check(unsafe { ffi::xl_write_string(self.handle, ptr, len) }) } /// Writes the next cell of the current row as an integer, or a blank cell for `None`. pub fn write_i64(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_int64(self.handle, v) }), - None => self.write_null(XL_T_I64), + Some(v) => check(unsafe { ffi::xl_write_int64(self.handle, v) }), + None => self.write_null(ColumnType::I64), } } @@ -107,32 +96,32 @@ impl WriterHandle { /// `None`. pub fn write_f64(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_float64(self.handle, v) }), - None => self.write_null(XL_T_F64), + Some(v) => check(unsafe { ffi::xl_write_float64(self.handle, v) }), + None => self.write_null(ColumnType::F64), } } /// Writes the next cell of the current row as a boolean, or a blank cell for `None`. pub fn write_bool(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_bool(self.handle, i32::from(v)) }), - None => self.write_null(XL_T_BOOL), + Some(v) => check(unsafe { ffi::xl_write_bool(self.handle, i32::from(v)) }), + None => self.write_null(ColumnType::Bool), } } /// Writes the next cell of the current row as a date, or a blank cell for `None`. pub fn write_date(&mut self, value: Option) -> Result<(), Error> { match value { - Some(v) => check(unsafe { crate::xl_write_date(self.handle, v.days_since_epoch) }), - None => self.write_null(XL_T_DATE), + Some(v) => check(unsafe { ffi::xl_write_date(self.handle, v.days_since_epoch) }), + None => self.write_null(ColumnType::Date), } } /// Writes the next cell of the current row as a time of day, or a blank cell for `None`. pub fn write_time(&mut self, value: Option