Exactly-once parallel CSV, run-time schemas in Rust and C++, safer bindings, coverage gates

.github/workflows/ci.yml

@@ -135,7 +135,22 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./TestResults/*.cobertura.xml
           flags: csharp
-          fail_ci_if_error: false
+          fail_ci_if_error: true
+
+      # The codecov/project checks are required on master, and Codecov posts them only for a commit it
+      # has heard about; when nothing here ran, an empty upload passes them on carried-forward coverage.
+      - name: Checkout for empty Codecov upload
+        if: env.RUN != 'true' && matrix.os == 'ubuntu-latest'
+        uses: actions/checkout@v7
+
+      - name: Empty Codecov upload
+        if: env.RUN != 'true' && matrix.os == 'ubuntu-latest'
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          run_command: empty-upload
+          force: true
+          fail_ci_if_error: true
 
   isa-fallback:
     name: Test ISA fallback (${{ matrix.isa }})

.github/workflows/fuzz.yml

@@ -53,7 +53,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target: [xlsx, xlsx-memory, xlsx-differential, xlsx-pipeline, xlsb, xlsb-memory, xlsb-differential, xlsb-pipeline, xls, encrypted, csv, csv-sniff, csv-parallel, inflate]
+        target: [xlsx, xlsx-memory, xlsx-differential, xlsx-pipeline, xlsb, xlsb-memory, xlsb-differential, xlsb-pipeline, xls, encrypted, csv, csv-sniff, csv-parallel, inflate, deflate, deflate-segments]
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -111,11 +111,12 @@ jobs:
             encrypted) dict=encrypted ;;
             csv*)      dict=csv ;;
             inflate)   dict=inflate ;;
+            deflate*)  dict= ;;
           esac
           ./libfuzzer-dotnet \
             --target_path=fuzz-out/ExcelReader.Fuzz \
             corpus/${{ matrix.target }} \
-            -dict="tests/ExcelReader.Fuzz/dictionaries/$dict.dict" \
+            ${dict:+-dict="tests/ExcelReader.Fuzz/dictionaries/$dict.dict"} \
             -max_total_time=${{ github.event.inputs.seconds || '600' }} \
             -max_len=131072 \
             -timeout=25 \

.github/workflows/native-bindings.yml

@@ -351,3 +351,24 @@ jobs:
 
       - name: Run the consumer
         run: ctest --test-dir build/consumer --output-on-failure -C Release
+
+      - name: Test with coverage and sanitizers (Linux)
+        if: runner.os == 'Linux'
+        env:
+          EXCELREADER_NATIVE_LIB: ${{ steps.lib.outputs.path }}
+          ASAN_OPTIONS: detect_leaks=0
+          UBSAN_OPTIONS: print_stacktrace=1
+        run: |
+          cmake -S cpp -B build/cpp-coverage -DEXCELREADER_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug "-DCMAKE_CXX_FLAGS=--coverage -fprofile-update=atomic -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer"
+          cmake --build build/cpp-coverage
+          ctest --test-dir build/cpp-coverage --output-on-failure
+          pipx run gcovr --root . --filter 'src/ExcelReader.Native/include/' --exclude-throw-branches --merge-lines --cobertura coverage-cpp.xml --txt
+
+      - name: Upload C++ coverage to Codecov
+        if: runner.os == 'Linux'
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage-cpp.xml
+          flags: cpp
+          fail_ci_if_error: false

ARCHITECTURE.md

@@ -175,15 +175,25 @@ the seam. A wrong start also shifts every field boundary, so on a source with qu
 discarded pass can hand the callback values that appear nowhere in the file. For an accumulator both are
 invisible: the discarded instance is thrown away.
 
-A guess is only wrong when the offset falls inside a quoted field. `CsvChunkWorker.GuessStart` first
-reads the byte before the chunk start and returns the chunk start unchanged when it already sits on a
-record boundary, because `CsvBoundaryResolver.FindRecordStart` only ever scans past a terminator and
+A guess can only be wrong when the offset falls inside a quoted field, so `CsvChunkWorker.GuessStart`
+first decides whether it does. `CsvBoundaryResolver.InferParity` reads the runs of quotes in a window
+of up to 64 KB from the chunk start, reaching back over a run the start cuts and looking past the
+chunk's end when it has to. Under RFC 4180 an odd run that opens a field follows a delimiter or a line
+start, and one that closes a field precedes a delimiter, a line end or the end of the source, so the
+first run that fits only one of the two decides. When a run fits both, the worker does not guess at
+all: the chunk is left to the merge, which reads it from its confirmed start.
+
+When the chunk starts outside quotes and the byte before it is a terminator, `GuessStart` returns the
+chunk start unchanged, because `CsvBoundaryResolver.FindRecordStart` only ever scans past a terminator and
 would otherwise overshoot by a whole record on every chunk that happens to be aligned — which, at a
-1 MiB minimum chunk size, is every chunk of a fixed-width file. That leaves quote-free sources exact,
-and the repeat documented as a caller obligation for the rest rather than designed out. Removing it
-there too needs a quote-parity prefix pass over the bytes before any parsing starts; that pass is a
-barrier and would cost roughly half the dop-16 runtime unless pipelined, which is more machinery than
-the hazard currently warrants.
+1 MiB minimum chunk size, is every chunk of a fixed-width file.
+
+`ForEachAsync` is therefore exactly once on a CSV that follows RFC 4180 quoting with no quoted field
+longer than 64 KB. A stray quote inside an unquoted field, text after a closing quote or a longer quoted
+field can still be misread, and those sources keep the repeat as a documented caller obligation.
+Removing it there too would need a quote-parity prefix pass over the bytes before any parsing starts.
+That pass is a barrier and would cost roughly half the dop-16 runtime unless pipelined, which is more
+machinery than sources that break RFC 4180 warrant.
 
 ## Shared plumbing
 

Directory.Build.props

@@ -27,7 +27,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Meziantou.Analyzer" Version="3.0.303">
+    <PackageReference Include="Meziantou.Analyzer" Version="3.0.315">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>

README.md

@@ -14,6 +14,12 @@ High-performance Excel reading and writing for .NET 10. Reads `.xlsx`, `.xlsb`,
 
 ExcelReader is built for streaming spreadsheet workloads where low allocations matter. It reads worksheet rows as lightweight `ref struct` values, resolves shared strings, recognizes date styles, and handles sparse cells. Workbooks open from a path, a `Stream`, or a caller-owned `ReadOnlyMemory<byte>` — the in-memory path decompresses parts directly, without a `ZipArchive` or an intermediate stream.
 
+It moves data, not presentation. Reading gives each cell's value and type, with a formula cell read as
+its last calculated result, plus sheet names, visibility and Excel tables. Formula text, merged cells,
+comments, hyperlinks, charts, images and conditional formatting are not read. Writing covers values,
+column widths and cell styles limited to bold, italic and a number format. To edit an existing workbook
+or produce formatted reports, use a library built for document manipulation.
+
 ## Install
 
 ```bash
@@ -45,7 +51,7 @@ foreach (Row row in workbook.FirstSheet)
 - [Reading untrusted files](docs/guide/untrusted-input.md) — the resource limits that bound what one hostile file can cost, and how to reject one cleanly.
 - [Migrating to v7](docs/guide/migrating-to-v7.md) — porting `CsvParallel` code to `SheetParallel`, which now reads XLSX and XLSB on several threads too.
 - [Migrating to v6](docs/guide/migrating-to-v6.md) — porting v5 reader code to workbooks and sheets, and reading sheets in parallel.
-- [Benchmarks](docs/performance/benchmarks.md) — throughput and allocation against Sep, Sylvan, OfficeIMO and SpreadCheetah. [Live results](https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/).
+- [Benchmarks](docs/performance/benchmarks.md) — throughput and allocation against Sep, Sylvan, ExcelDataReader, OfficeIMO and SpreadCheetah. [Live results](https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/).
 
 ## Command line
 

codecov.yml

@@ -1,3 +1,30 @@
+coverage:
+  status:
+    project:
+      csharp:
+        target: 88%
+        flags:
+          - csharp
+      python:
+        target: 88%
+        flags:
+          - python
+      rust:
+        target: 88%
+        flags:
+          - rust
+      cpp:
+        target: 88%
+        flags:
+          - cpp
+    patch:
+      default:
+        target: 88%
+
+# gcov reports the C++ headers through the cpp/include/xl symlink; git tracks them at their real path.
+fixes:
+  - "cpp/include/xl/::src/ExcelReader.Native/include/"
+
 # Each language uploads under its own flag from a path-filtered job, so a flag a commit did not run keeps its last report.
 flag_management:
   default_rules:
@@ -12,3 +39,6 @@ flag_management:
     - name: python
       paths:
         - python/
+    - name: cpp
+      paths:
+        - src/ExcelReader.Native/include/

cpp/README.md

@@ -106,8 +106,23 @@ for (xl::Sheet sheet : *workbook->sheets()) { std::println("{}", sheet.name().va
 for (const auto& column : *workbook->sheet(1).infer_schema(1, 100)) {
     // column.name is nullopt when the column must be resolved by column.index instead.
 }
+
+// With no row type, parse the sheet through that schema. The columns are the xl::ColumnRef
+// write_columns takes, so the table writes back as it was read.
+auto schema = workbook->sheet(1).infer_schema();
+auto table = xl::parse(workbook->sheet(1), *schema);
+for (const xl::ColumnRef& column : table->columns()) { /* column.type, column.values, ... */ }
+auto written = xl::write_columns("copy.xlsx", table->columns());
+
+// Or a batch at a time: an xl::TableReader, whose batches are xl::Tables.
+auto reader = xl::parse_batches(workbook->sheet(1), *schema, {.batch_size = 10'000});
+for (auto& batch : *reader) { if (!batch) { break; } /* batch->columns() */ }
 ```
 
+A column matched by index has no name, and writing needs every column named or none. The schema is
+a guess over a sample: a column inferred as `XL_T_I64` that holds a fraction further down fails the
+read, unless the column is `nullable`.
+
 Every entry point returns `std::expected<T, xl::Error>` — this header throws nothing.
 
 A workbook hands out sheets. An `xl::Sheet` is the workbook handle and an index: it owns nothing,
@@ -299,6 +314,18 @@ while (true)
 
 Same `header_row`/`batch_size` meaning as `xl::parse_batches` above.
 
+With no row type, both take the columns at run time instead, such as the ones `infer_schema`
+returned. A column with a name is matched by it, one without by its index:
+
+```cpp
+auto schema = workbook->sheet(0).infer_schema();
+auto table = xl::to_arrow(workbook->sheet(0), *schema);
+auto stream = xl::to_arrow_stream(workbook->sheet(0), *schema, {.batch_size = 10'000});
+```
+
+The schema is a guess over a sample: a column inferred as `XL_T_I64` that holds a fraction further
+down fails the read, unless the column is `nullable`.
+
 ## Writing
 
 Two layers, mirroring the two on the reading side.

cpp/tests/aggregate.cpp

@@ -473,6 +473,34 @@ static int test_sheet_aggregate_reports_a_throwing_accumulate()
     return 0;
 }
 
+static int test_sheet_aggregate_fails_on_a_sheet_past_the_end()
+{
+    const auto bytes = generate(XL_FORMAT_XLSX, 10);
+    auto workbook = open_generated(bytes, XL_FORMAT_XLSX);
+    CHECK(workbook.has_value(), "open the generated workbook");
+    auto result = workbook->sheet(99).aggregate(RowCounter{}, 1, 1);
+    CHECK(!result.has_value() && result.error().code < 0, "a sheet past the end fails with the library's status");
+    return 0;
+}
+
+static int test_aggregate_reports_a_seed_copy_that_throws()
+{
+    struct ThrowsOnCopy
+    {
+        ThrowsOnCopy() = default;
+        ThrowsOnCopy(const ThrowsOnCopy &) { throw std::runtime_error("no copies"); }
+        ThrowsOnCopy(ThrowsOnCopy &&) noexcept = default;
+        int32_t accumulate(xl::RowView) noexcept { return XL_OK; }
+        int32_t combine(ThrowsOnCopy &) noexcept { return XL_OK; }
+    };
+
+    std::span<const uint8_t> data(reinterpret_cast<const uint8_t *>(kCsv.data()), kCsv.size());
+    auto result = xl::aggregate_csv(data, ThrowsOnCopy{});
+    CHECK(!result.has_value() && std::string_view(result.error().message).find("threw") != std::string_view::npos,
+          "a seed copy that throws must be reported as a callback that threw");
+    return 0;
+}
+
 static int test_existing_csv_accumulator_works_unchanged()
 {
     const auto bytes = generate(XL_FORMAT_CSV, 10'000);
@@ -508,6 +536,14 @@ int main()
     {
         return failed;
     }
+    if (int failed = test_sheet_aggregate_fails_on_a_sheet_past_the_end())
+    {
+        return failed;
+    }
+    if (int failed = test_aggregate_reports_a_seed_copy_that_throws())
+    {
+        return failed;
+    }
     if (int failed = test_aggregate_csv_memory())
     {
         return failed;

cpp/tests/arrow.cpp

@@ -93,6 +93,20 @@ int main()
 
         auto second = xl::to_arrow_stream<Record>(stream_workbook->sheet(0), {.batch_size = 8});
         CHECK(second.has_value(), "a second stream on one workbook opens");
+
+        xl::ArrowStream moved = std::move(*second);
+        CHECK(!second->schema().has_value(), "a moved-from stream has no schema");
+        CHECK(!second->next().has_value(), "a moved-from stream reads nothing");
+        CHECK(moved.schema().has_value(), "the moved-to stream still reports its schema");
+
+        const xl::InferredColumn columns[] = {{"Coluna3", 0, XL_T_I64, false}};
+        CHECK(!xl::to_arrow_stream<Record>(stream_workbook->sheet(99)).has_value(),
+              "a stream over a sheet past the end fails to open");
+        CHECK(!xl::to_arrow_stream(stream_workbook->sheet(99), columns).has_value(),
+              "a runtime-schema stream over a sheet past the end fails to open");
+        auto zero_batch = xl::to_arrow_stream(stream_workbook->sheet(0), columns, {.batch_size = 0});
+        CHECK(!zero_batch.has_value() && zero_batch.error().code == XL_INVALID_ARGUMENT,
+              "a runtime-schema stream rejects a batch size below 1");
     }
 
     {
@@ -199,6 +213,43 @@ int main()
         CHECK(reader.has_value(), "a typed reader opens while an Arrow stream is live");
     }
 
+    {
+        auto wb = xl::Workbook::open(EXCELREADER_FIXTURE_PATH, options);
+        CHECK(wb.has_value(), "opening a workbook for the inferred-schema test must succeed");
+        auto schema = wb->sheet(0).infer_schema();
+        CHECK(schema.has_value() && !schema->empty(), "infer_schema must find columns");
+
+        auto inferred = xl::to_arrow(wb->sheet(0), *schema);
+        CHECK(inferred.has_value(), "to_arrow over an inferred schema must succeed");
+        CHECK(inferred->schema.n_children == static_cast<int64_t>(schema->size()), "one Arrow column per inferred column");
+        CHECK(inferred->array.length == 100, "the inferred read holds every data row");
+        CHECK(std::strcmp(inferred->schema.children[0]->name, "Coluna1") == 0, "columns keep their header names");
+
+        auto by_index = wb->sheet(0).infer_schema(0);
+        CHECK(by_index.has_value() && !(*by_index)[0].name.has_value(), "a schema without a header has no names");
+        auto unnamed = xl::to_arrow(wb->sheet(0), *by_index, {.header_row = 0});
+        CHECK(unnamed.has_value(), "to_arrow resolves unnamed columns by index");
+        CHECK(unnamed->array.length == 101, "with no header the header row is data");
+
+        auto empty = xl::to_arrow(wb->sheet(0), std::span<const xl::InferredColumn>{});
+        CHECK(!empty.has_value() && empty.error().code == XL_INVALID_ARGUMENT, "an empty schema is rejected");
+
+        auto stream = xl::to_arrow_stream(wb->sheet(0), *schema, {.batch_size = 8});
+        CHECK(stream.has_value(), "to_arrow_stream over an inferred schema must open");
+        int64_t rows = 0;
+        while (true)
+        {
+            auto batch = stream->next();
+            CHECK(batch.has_value(), "an inferred-schema batch must read without error");
+            if (!batch->has_value())
+            {
+                break;
+            }
+            rows += (*batch)->array.length;
+        }
+        CHECK(rows == 100, "the inferred stream delivers every data row");
+    }
+
     std::printf("OK: C++ arrow test passed\n");
     return 0;
 }