From 42e904b91a273801f391b0adf9d7cf39fc2776cf Mon Sep 17 00:00:00 2001 From: "alex.bianchi" Date: Tue, 17 Nov 2026 02:56:52 +0000 Subject: [PATCH] Reuse planned file metrics for eager Iceberg column statistics Request column statistics from the scan (iceberg-rust with_column_stats, apache/iceberg-rust#3424) and summarize them from the planned FileScanTasks instead of re-reading manifests, avoiding double IO. Bump iceberg-rust to 13f84f9 (Arrow 60 plus #3424), patching datafusion-iceberg to the same revision so FileScanTask is one type, and adapt to its API (UnboundPartitionField builder, FileScanTaskDeleteFile accessors). Adapt the column byte statistics tests to DataFusion 56. --- Cargo.lock | 223 +++++++++++++++--- Cargo.toml | 8 + .../distributed-iceberg-test-tools/Cargo.toml | 2 +- .../src/table.rs | 13 +- datafusion/distributed-iceberg/Cargo.toml | 8 +- datafusion/distributed-iceberg/README.md | 6 +- datafusion/distributed-iceberg/src/config.rs | 2 +- .../src/eager/planned_files.rs | 40 +++- .../src/table_provider/column_statistics.rs | 166 +++++++------ .../src/table_provider/scan.rs | 62 +++-- .../tests/shared_reader.rs | 2 +- .../statistics/column_byte_statistics.rs | 55 +++-- .../distributed/src/events/route_tasks.rs | 1 + 13 files changed, 408 insertions(+), 180 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e076a7b81..d09f2b8d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "3.0.0" @@ -178,26 +184,26 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "apache-avro" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" dependencies = [ "bigdecimal", "bon", "crc32fast", - "digest 0.10.7", + "digest 0.11.3", "log", - "miniz_oxide 0.8.9", + "miniz_oxide", "num-bigint 0.4.8", + "ouroboros", "quad-rand", - "rand 0.9.5", + "rand 0.10.3", "regex-lite", "serde", "serde_bytes", "serde_json", "snap", - "strum 0.27.2", - "strum_macros 0.27.2", + "strum 0.28.0", "thiserror 2.0.21", "uuid", "zstd 0.13.5", @@ -1079,7 +1085,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.9.1", + "miniz_oxide", "object 0.40.0", "rustc-demangle", "windows-link", @@ -3209,12 +3215,6 @@ dependencies = [ "syn 3.0.10", ] -[[package]] -name = "dissimilar" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" - [[package]] name = "dlv-list" version = "0.5.2" @@ -3374,16 +3374,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "expect-test" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" -dependencies = [ - "dissimilar", - "once_cell", -] - [[package]] name = "eyre" version = "0.6.14" @@ -3475,7 +3465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" dependencies = [ "crc32fast", - "miniz_oxide 0.9.1", + "miniz_oxide", "zlib-rs", ] @@ -3625,6 +3615,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777d18aa0f12f8b285331cd867133ee14422b3f023f6d388034c47d43e28786a" +dependencies = [ + "approx", + "num-traits", + "serde", + "thiserror 2.0.21", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -3782,6 +3793,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -4016,7 +4033,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust.git?rev=0d5eb123495f765ea151e78c8964919d554d4b98#0d5eb123495f765ea151e78c8964919d554d4b98" +source = "git+https://github.com/DataDog/iceberg-rust.git?rev=13f84f93f1a09619f473b788b2a88ef1da0c7102#13f84f93f1a09619f473b788b2a88ef1da0c7102" dependencies = [ "aes-gcm", "anyhow", @@ -4024,6 +4041,7 @@ dependencies = [ "array-init", "arrow-arith", "arrow-array", + "arrow-avro", "arrow-buffer", "arrow-cast", "arrow-ord", @@ -4039,7 +4057,6 @@ dependencies = [ "chrono", "crc32fast", "derive_builder", - "expect-test", "fastnum", "flate2", "fnv", @@ -4052,8 +4069,9 @@ dependencies = [ "once_cell", "ordered-float", "parquet", + "parquet-geospatial", "rand 0.9.5", - "reqwest 0.12.28", + "reqwest 0.13.5", "roaring", "serde", "serde_bytes", @@ -4093,7 +4111,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust.git?rev=0d5eb123495f765ea151e78c8964919d554d4b98#0d5eb123495f765ea151e78c8964919d554d4b98" +source = "git+https://github.com/DataDog/iceberg-rust.git?rev=13f84f93f1a09619f473b788b2a88ef1da0c7102#13f84f93f1a09619f473b788b2a88ef1da0c7102" dependencies = [ "proc-macro2", "quote", @@ -4103,7 +4121,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust.git?rev=0d5eb123495f765ea151e78c8964919d554d4b98#0d5eb123495f765ea151e78c8964919d554d4b98" +source = "git+https://github.com/DataDog/iceberg-rust.git?rev=13f84f93f1a09619f473b788b2a88ef1da0c7102#13f84f93f1a09619f473b788b2a88ef1da0c7102" dependencies = [ "anyhow", "async-trait", @@ -4111,6 +4129,7 @@ dependencies = [ "cfg-if", "futures", "iceberg", + "iceberg-property-macro", "opendal", "reqsign-aws-v4", "reqsign-core", @@ -4833,15 +4852,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - [[package]] name = "miniz_oxide" version = "0.9.1" @@ -5058,6 +5068,28 @@ dependencies = [ "libm", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num_threads" version = "0.1.8" @@ -5386,6 +5418,30 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + [[package]] name = "outref" version = "0.5.2" @@ -5508,6 +5564,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "parquet-geospatial", "ring", "seq-macro", "simdutf8", @@ -5517,6 +5574,19 @@ dependencies = [ "zstd 0.14.4", ] +[[package]] +name = "parquet-geospatial" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eef66af500dd4e6f546d88ed582277559f4dba847ad0f1388c118ea27908fddb" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -5858,6 +5928,15 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -5891,6 +5970,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + [[package]] name = "prost" version = "0.14.6" @@ -6510,6 +6602,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -7720,6 +7814,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.19+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.14.9" @@ -8531,12 +8655,33 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wkb" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04ab2f2845abe967a213998446345a1a6fc9a8f318cd88f668a8e507861d13bc" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index 57d054e7a..edadf77dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,11 @@ datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "16 [workspace.lints.clippy] disallowed_types = "deny" disallowed-methods = "deny" + +# datafusion-iceberg pins apache/iceberg-rust at an older Arrow 60 revision. Resolve it to the +# same revision as this workspace (apache/iceberg-rust 13f84f9: that revision plus FileScanTask +# column statistics, #3424) so FileScanTask is one type. Drop this once datafusion-iceberg +# depends on a revision with both. +[patch."https://github.com/apache/iceberg-rust.git"] +iceberg = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" } diff --git a/datafusion/distributed-iceberg-test-tools/Cargo.toml b/datafusion/distributed-iceberg-test-tools/Cargo.toml index 30792043d..dbb9447d8 100644 --- a/datafusion/distributed-iceberg-test-tools/Cargo.toml +++ b/datafusion/distributed-iceberg-test-tools/Cargo.toml @@ -15,7 +15,7 @@ datafusion = { workspace = true, features = ["parquet", "sql"] } datafusion-distributed = { path = "../distributed", features = ["integration"] } datafusion-proto = { workspace = true } # Match datafusion-distributed-iceberg's iceberg-rust revision so tables share one type. -iceberg = { git = "https://github.com/apache/iceberg-rust.git", rev = "0d5eb123495f765ea151e78c8964919d554d4b98" } +iceberg = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" } rand = { version = "0.10", default-features = false, features = ["alloc"] } rand_pcg = "0.10" tempfile = "3.18" diff --git a/datafusion/distributed-iceberg-test-tools/src/table.rs b/datafusion/distributed-iceberg-test-tools/src/table.rs index 103176aba..5e5f32c58 100644 --- a/datafusion/distributed-iceberg-test-tools/src/table.rs +++ b/datafusion/distributed-iceberg-test-tools/src/table.rs @@ -7,7 +7,9 @@ use datafusion::common::exec_datafusion_err; use datafusion::error::Result; use iceberg::arrow::schema_to_arrow_schema; use iceberg::memory::MemoryCatalog; -use iceberg::spec::{Schema, TableMetadata, Transform, UnboundPartitionSpec}; +use iceberg::spec::{ + Schema, TableMetadata, Transform, UnboundPartitionField, UnboundPartitionSpec, +}; use iceberg::table::Table; use iceberg::{Catalog, MetadataLocation, TableCreation, TableIdent}; use tempfile::TempDir; @@ -70,9 +72,12 @@ impl TableBuilder { })?; partition_spec = partition_spec .add_partition_field( - source.id, - partition_field_name(&column, transform), - transform, + UnboundPartitionField::builder() + .source_ids(vec![source.id]) + .name(partition_field_name(&column, transform)) + .transform(transform) + .build() + .map_err(df_err)?, ) .map_err(df_err)?; } diff --git a/datafusion/distributed-iceberg/Cargo.toml b/datafusion/distributed-iceberg/Cargo.toml index 27659aef0..6f1953b4e 100644 --- a/datafusion/distributed-iceberg/Cargo.toml +++ b/datafusion/distributed-iceberg/Cargo.toml @@ -12,11 +12,13 @@ async-trait = "0.1" bytes = "1" datafusion = { workspace = true, features = ["sql"] } datafusion-proto = { workspace = true } -# Match datafusion-iceberg's iceberg-rust revision so FileScanTask has one shared type. +# apache/iceberg-rust 13f84f9: Arrow 60 plus FileScanTask column statistics (#3424). The commit is +# fetched from DataDog/iceberg-rust, which mirrors upstream; the workspace [patch] points +# datafusion-iceberg at it too so FileScanTask is one type. iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg.git", rev = "5398be196e47ef42543f6e9c72d15cad898e5143" } futures = "0.3" -iceberg = { git = "https://github.com/apache/iceberg-rust.git", rev = "0d5eb123495f765ea151e78c8964919d554d4b98" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust.git", rev = "0d5eb123495f765ea151e78c8964919d554d4b98", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } +iceberg = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" } +iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] } datafusion-distributed = { path = "../distributed", features = ["sql"] } delegate = "0.13" prost = "0.14.1" diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 30b773af8..dfe7f6bbd 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -101,8 +101,10 @@ Snapshot distinct-count estimates are retained as an inexact fallback, capped by selected-file row counts when known. They are not recomputed for the selected files and can overestimate the filtered result. Sum statistics remain unknown. -Another manifest pass is needed because the pinned Iceberg API does not retain -column metrics in file tasks. Scan task counts continue to use whole-file bytes. +The metrics come from the planned file tasks themselves: the scan asks Iceberg to +retain the statistics columns' metrics (`TableScanBuilder::with_column_stats`, from +DataDog's iceberg-rust fork), so no manifest is read a second time. They are dropped +from the tasks once summarized. Scan task counts continue to use whole-file bytes. This integration branch also exports `datafusion_distributed::ColumnByteStatisticsProvider`. To preserve available column-byte estimates through grouping keys and inner joins, diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index ee9df994c..f1316a691 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -14,7 +14,7 @@ extensions_options! { pub row_group_filtering_enabled: bool, default = true /// Whether to apply row-level selections while reading Parquet files. pub row_selection_enabled: bool, default = false - /// Read column statistics during planning (may require another manifest pass). + /// Read column statistics during planning (retained from the files selected during planning). /// Eager scans aggregate selected-file metrics and retain inexact snapshot NDV. pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. diff --git a/datafusion/distributed-iceberg/src/eager/planned_files.rs b/datafusion/distributed-iceberg/src/eager/planned_files.rs index a4c5c0d67..d9f188b01 100644 --- a/datafusion/distributed-iceberg/src/eager/planned_files.rs +++ b/datafusion/distributed-iceberg/src/eager/planned_files.rs @@ -6,7 +6,8 @@ use datafusion::common::{Result, Statistics}; use datafusion::execution::TaskContext; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; use futures::TryStreamExt; -use iceberg::scan::{FileScanTask, TableScan}; +use iceberg::scan::{FileScanTask, FileScanTaskColumnStats, TableScan}; +use iceberg::spec::{Datum, PrimitiveLiteral}; use crate::common::df_err; @@ -86,12 +87,41 @@ fn retained_size(task: &FileScanTask) -> usize { + size_of_val(task.project_field_ids()) + size_of_val(task.deletes()) + task.key_metadata().map_or(0, <[u8]>::len); + if let Some(stats) = task.column_stats() { + bytes = bytes.saturating_add(column_stats_size(stats)); + } for delete in task.deletes() { bytes = bytes - .saturating_add(delete.file_path.len()) - .saturating_add(delete.referenced_data_file.as_deref().map_or(0, str::len)) - .saturating_add(delete.equality_ids.as_deref().map_or(0, size_of_val)) - .saturating_add(delete.key_metadata.as_deref().map_or(0, <[u8]>::len)); + .saturating_add(delete.file_path().len()) + .saturating_add(delete.referenced_data_file().map_or(0, str::len)) + .saturating_add(delete.equality_ids().map_or(0, size_of_val)) + .saturating_add(delete.key_metadata().map_or(0, <[u8]>::len)); } bytes } + +/// Approximate heap size of the metrics a task retains for column statistics. +fn column_stats_size(stats: &FileScanTaskColumnStats) -> usize { + let counts = [ + stats.column_sizes(), + stats.value_counts(), + stats.null_value_counts(), + stats.nan_value_counts(), + ] + .iter() + .map(|counts| counts.len() * size_of::<(i32, u64)>()) + .sum::(); + let bounds = [stats.lower_bounds(), stats.upper_bounds()] + .iter() + .flat_map(|bounds| bounds.values()) + .map(|bound| { + size_of::<(i32, Datum)>() + + match bound.literal() { + PrimitiveLiteral::String(value) => value.len(), + PrimitiveLiteral::Binary(value) => value.len(), + _ => 0, + } + }) + .sum::(); + size_of::() + counts + bounds +} diff --git a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs index 2e51afe54..fc5b97de1 100644 --- a/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs +++ b/datafusion/distributed-iceberg/src/table_provider/column_statistics.rs @@ -1,23 +1,48 @@ use crate::common::df_err; use datafusion::common::runtime::JoinSet; use datafusion::common::stats::Precision; -use datafusion::common::{ColumnStatistics, HashMap, HashSet, Result, exec_err}; +use datafusion::common::{ColumnStatistics, HashMap, Result, exec_err}; use datafusion::error::DataFusionError; use datafusion::scalar::ScalarValue; use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1; use iceberg::spec::{ DataFile, Datum, ManifestContentType, ManifestList, NestedFieldRef, PrimitiveLiteral, - PrimitiveType, SnapshotRef, Type, + PrimitiveType, Schema, SnapshotRef, Type, }; use iceberg::table::Table; use std::sync::Arc; -use iceberg::scan::FileScanTask; +use iceberg::scan::{FileScanTask, FileScanTaskColumnStats}; + +/// Names of the primitive leaf columns under `fields`, in schema-dotted form. Passed to +/// `TableScanBuilder::with_column_stats` so each planned `FileScanTask` carries the file's +/// metrics for exactly the columns these statistics cover. Leaves that are not primitive +/// (variants) have no file metrics and are skipped. +pub(crate) fn column_stats_names( + schema: &Schema, + fields: &[NestedFieldRef], +) -> Result> { + let mut names = vec![]; + for field in fields { + let mut leaves = vec![]; + leaves_of(field, &mut leaves); + for leaf in leaves { + if !matches!(leaf.field_type.as_ref(), Type::Primitive(_)) { + continue; + } + let Some(name) = schema.name_by_field_id(leaf.id) else { + return exec_err!("Iceberg column {} has no name in the scan schema", leaf.id); + }; + names.push(name.to_string()); + } + } + Ok(names) +} /// Selected-file metrics, with snapshot NDV retained only as an inexact fallback. -/// FileScanTask does not retain manifest metrics, so this requires a metadata-only -/// second pass. Stop as soon as all selected files have been found. -pub(crate) async fn selected_column_statistics( +/// Metrics come from the `FileScanTask::column_stats` that the scan retained while +/// planning (see [`column_stats_names`]), so no manifest is read a second time. +pub(crate) fn selected_column_statistics( table: &Table, snapshot: &SnapshotRef, fields: &[NestedFieldRef], @@ -42,10 +67,6 @@ pub(crate) async fn selected_column_statistics( // Full-file metrics cannot describe a byte-range task. return Ok(vec![ColumnStatistics::new_unknown(); fields.len()]); } - let mut remaining: HashSet<_> = files.iter().map(|task| task.data_file_path()).collect(); - if remaining.len() != files.len() { - return exec_err!("Duplicate whole-file tasks in Iceberg column statistics lookup"); - } let field_ids: Vec<_> = fields .iter() .map(|field| { @@ -55,51 +76,31 @@ pub(crate) async fn selected_column_statistics( }) .collect(); let mut statistics = vec![None; fields.len()]; - let manifests = table - .manifest_list_reader(snapshot) - .load() - .await - .map_err(df_err)?; - for manifest in manifests - .entries() - .iter() - .filter(|manifest| manifest.content == ManifestContentType::Data) - { - let manifest = table - .manifest_reader() - .read(manifest) - .await - .map_err(df_err)?; - for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { - let file = entry.data_file(); - if !remaining.remove(file.file_path()) { - continue; - } - for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) { - // Leaf metrics cannot describe a parent struct/list/map's value or null count. - let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) { - data_file_col_stats(file, field.id) - } else { - ColumnStatistics::new_unknown() - }; - next.byte_size = ids - .iter() - .try_fold(0_usize, |total, id| { - let size = usize::try_from(*file.column_sizes().get(id)?).ok()?; - total.checked_add(size) - }) - .map(Precision::Inexact) - .unwrap_or(Precision::Absent); - *acc = Some(merge_col_stats(acc.take(), next)); - } - } - if remaining.is_empty() { - break; + for task in files { + let Some(file) = task.column_stats() else { + return exec_err!( + "Planned Iceberg file {} carries no column statistics", + task.data_file_path() + ); + }; + for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) { + // Leaf metrics cannot describe a parent struct/list/map's value or null count. + let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) { + task_col_stats(file, field.id) + } else { + ColumnStatistics::new_unknown() + }; + next.byte_size = ids + .iter() + .try_fold(0_usize, |total, id| { + let size = usize::try_from(*file.column_sizes().get(id)?).ok()?; + total.checked_add(size) + }) + .map(Precision::Inexact) + .unwrap_or(Precision::Absent); + *acc = Some(merge_col_stats(acc.take(), next)); } } - if !remaining.is_empty() { - return exec_err!("Planned Iceberg files missing from snapshot manifests"); - } let rows = files.iter().try_fold(0_usize, |rows, task| { rows.checked_add(usize::try_from(task.record_count()?).ok()?) }); @@ -123,17 +124,23 @@ pub(crate) async fn selected_column_statistics( } fn leaf_ids(field: &NestedFieldRef, ids: &mut Vec) { + let mut leaves = vec![]; + leaves_of(field, &mut leaves); + ids.extend(leaves.iter().map(|leaf| leaf.id)); +} + +fn leaves_of<'a>(field: &'a NestedFieldRef, leaves: &mut Vec<&'a NestedFieldRef>) { match field.field_type.as_ref() { - Type::Primitive(_) | Type::Variant(_) => ids.push(field.id), + Type::Primitive(_) | Type::Variant(_) => leaves.push(field), Type::Struct(fields) => { for field in fields.fields() { - leaf_ids(field, ids); + leaves_of(field, leaves); } } - Type::List(list) => leaf_ids(&list.element_field, ids), + Type::List(list) => leaves_of(&list.element_field, leaves), Type::Map(map) => { - leaf_ids(&map.key_field, ids); - leaf_ids(&map.value_field, ids); + leaves_of(&map.key_field, leaves); + leaves_of(&map.value_field, leaves); } } } @@ -279,28 +286,43 @@ fn merge_col_stats(acc: Option, next: ColumnStatistics) -> Col } fn data_file_col_stats(df: &DataFile, id: i32) -> ColumnStatistics { + file_col_stats( + df.null_value_counts().get(&id).copied(), + df.lower_bounds().get(&id), + df.upper_bounds().get(&id), + df.column_sizes().get(&id).copied(), + ) +} + +fn task_col_stats(stats: &FileScanTaskColumnStats, id: i32) -> ColumnStatistics { + file_col_stats( + stats.null_value_counts().get(&id).copied(), + stats.lower_bounds().get(&id), + stats.upper_bounds().get(&id), + stats.column_sizes().get(&id).copied(), + ) +} + +fn file_col_stats( + null_count: Option, + lower_bound: Option<&Datum>, + upper_bound: Option<&Datum>, + column_size: Option, +) -> ColumnStatistics { ColumnStatistics { - null_count: df - .null_value_counts() - .get(&id) - .map(|n| Precision::Exact(*n as usize)) + null_count: null_count + .map(|n| Precision::Exact(n as usize)) .unwrap_or(Precision::Absent), - min_value: df - .lower_bounds() - .get(&id) + min_value: lower_bound .and_then(datum_to_scalar) .map(Precision::Inexact) .unwrap_or(Precision::Absent), - max_value: df - .upper_bounds() - .get(&id) + max_value: upper_bound .and_then(datum_to_scalar) .map(Precision::Inexact) .unwrap_or(Precision::Absent), - byte_size: df - .column_sizes() - .get(&id) - .map(|n| Precision::Inexact(*n as usize)) + byte_size: column_size + .map(|n| Precision::Inexact(n as usize)) .unwrap_or(Precision::Absent), sum_value: Precision::Absent, distinct_count: Precision::Absent, diff --git a/datafusion/distributed-iceberg/src/table_provider/scan.rs b/datafusion/distributed-iceberg/src/table_provider/scan.rs index 59c127123..959f4c4c0 100644 --- a/datafusion/distributed-iceberg/src/table_provider/scan.rs +++ b/datafusion/distributed-iceberg/src/table_provider/scan.rs @@ -2,7 +2,9 @@ use crate::common::df_err; use crate::common::scan_config::{ScanConfig, apply_column_statistics}; use crate::eager::assign_files; use crate::eager::planned_files::PlannedFiles; -use crate::table_provider::column_statistics::{compute_column_stats, selected_column_statistics}; +use crate::table_provider::column_statistics::{ + column_stats_names, compute_column_stats, selected_column_statistics, +}; use crate::{EagerIcebergDataSource, IcebergConfig, IcebergWorkUnitFeed, LazyIcebergDataSource}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, config_err, exec_err}; @@ -60,6 +62,25 @@ pub(crate) async fn plan_scan( .unwrap_or_else(iceberg::Runtime::current), &context, ); + // The columns whose statistics are requested, with the schema that names them. + let stats_fields = if IcebergConfig::from_task_context(&context).column_stats_enabled { + let schema = match &table_snapshot { + Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, + // empty table + None => table.metadata().current_schema().clone(), + }; + let fields = schema.as_struct().fields(); + let fields: Vec<_> = match opts.projection { + Some(projection) => projection + .iter() + .map(|&idx| Arc::clone(&fields[idx])) + .collect(), + None => fields.to_vec(), + }; + Some((schema, fields)) + } else { + None + }; let planned = if IcebergConfig::from_task_context(&context).eager_file_planning { if partitioning.partition_count() == 0 { return config_err!("Iceberg scan requires at least one output partition"); @@ -74,6 +95,10 @@ pub(crate) async fn plan_scan( if let Some(predicates) = &predicates { builder = builder.with_filter(predicates.clone()); } + if let Some((schema, fields)) = &stats_fields { + // Keep each file's metrics on its task, so statistics need no second manifest pass. + builder = builder.with_column_stats(column_stats_names(schema, fields)?); + } let table_scan = builder.build().map_err(df_err)?; // File discovery runs on Iceberg's I/O runtime and is canceled if planning is dropped. let (_cancel_on_drop, canceled) = oneshot::channel::<()>(); @@ -84,7 +109,7 @@ pub(crate) async fn plan_scan( planned = PlannedFiles::collect(table_scan, &planning_context) => planned, } }); - Some(Arc::new(task.await.map_err(df_err)??)) + Some(task.await.map_err(df_err)??) } else { None }; @@ -92,33 +117,17 @@ pub(crate) async fn plan_scan( || planned .as_ref() .is_some_and(|files| files.tasks().iter().any(|task| !task.deletes().is_empty())); - let column_stats = if IcebergConfig::from_task_context(&context).column_stats_enabled { - let schema = match &table_snapshot { - Some(snap) => snap.schema(table.metadata()).map_err(df_err)?, - // empty table - None => table.metadata().current_schema().clone(), - }; - let fields = schema.as_struct().fields(); - let fields: Vec<_> = match opts.projection { - Some(projection) => projection - .iter() - .map(|&idx| Arc::clone(&fields[idx])) - .collect(), - None => fields.to_vec(), - }; + let column_stats = if let Some((_, fields)) = &stats_fields { if let Some(files) = &planned && let Some(snapshot) = &table_snapshot { - Some( - selected_column_statistics( - &table, - snapshot, - &fields, - files.tasks(), - may_filter_rows, - ) - .await?, - ) + Some(selected_column_statistics( + &table, + snapshot, + fields, + files.tasks(), + may_filter_rows, + )?) } else { let field_ids = fields.iter().map(|field| field.id).collect(); Some(compute_column_stats(table.clone(), field_ids, table_snapshot.clone()).await?) @@ -127,6 +136,7 @@ pub(crate) async fn plan_scan( None }; if let Some(planned) = planned { + let planned = Arc::new(planned); let mut statistics = planned.statistics(&scan.schema, may_filter_rows); if let Some(column_stats) = column_stats { apply_column_statistics(&mut statistics, column_stats); diff --git a/datafusion/distributed-iceberg/tests/shared_reader.rs b/datafusion/distributed-iceberg/tests/shared_reader.rs index 2b8701899..ca777cbd7 100644 --- a/datafusion/distributed-iceberg/tests/shared_reader.rs +++ b/datafusion/distributed-iceberg/tests/shared_reader.rs @@ -106,7 +106,7 @@ mod tests { .with_file_type(DataContentType::PositionDeletes) .with_file_format(DataFileFormat::Parquet) .with_partition_spec_id(0) - .build(); + .build()?; let mut groups = Vec::new(); for path in paths { let size = fs::metadata(&path)?.len(); diff --git a/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs b/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs index 5981f1dd9..54fc171bc 100644 --- a/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs +++ b/datafusion/distributed/src/distributed_planner/statistics/column_byte_statistics.rs @@ -139,12 +139,19 @@ mod tests { use datafusion::physical_plan::test::exec::StatisticsExec; use datafusion::prelude::SessionConfig; + fn registry_stats(plan: &dyn ExecutionPlan) -> Result { + Ok(StatisticsContext::new_with_registry(registry()) + .compute(plan, &StatisticsArgs::new())? + .as_ref() + .clone()) + } + #[test] fn group_keys_keep_bytes_without_retaining_payload_cost() -> Result<()> { let aggregate = distinct_keys(source(10_000, 32, true))?; let ordinary = StatisticsContext::new().compute(aggregate.as_ref(), &StatisticsArgs::new())?; - let stats = registry().compute_base(aggregate.as_ref())?; + let stats = registry_stats(aggregate.as_ref())?; assert_eq!(stats.num_rows, ordinary.num_rows); assert_eq!( stats.column_statistics[0].distinct_count, @@ -176,25 +183,24 @@ mod tests { NullEquality::NullEqualsNothing, false, )?) as Arc; - for (enabled, expected) in [(false, JoinType::LeftSemi), (true, JoinType::RightSemi)] { - let mut config = SessionConfig::new(); - config.options_mut().optimizer.use_statistics_registry = enabled; - let state = SessionStateBuilder::new() - .with_default_features() - .with_config(config) - .with_statistics_registry(registry()) - .build(); - let optimized = - JoinSelection::new().optimize_with_context(Arc::clone(&join), &state)?; - assert_eq!( - optimized - .downcast_ref::() - .unwrap() - .join_type(), - &expected - ); - assert_eq!(optimized.schema(), join.schema()); - } + // DataFusion 56's own join selection already builds the derived keys here, so only the + // provider-enabled outcome is asserted; the disabled baseline no longer discriminates. + let mut config = SessionConfig::new(); + config.options_mut().optimizer.use_statistics_registry = true; + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_statistics_registry(registry()) + .build(); + let optimized = JoinSelection::new().optimize_with_context(Arc::clone(&join), &state)?; + assert_eq!( + optimized + .downcast_ref::() + .unwrap() + .join_type(), + &JoinType::RightSemi + ); + assert_eq!(optimized.schema(), join.schema()); Ok(()) } @@ -203,11 +209,11 @@ mod tests { let input = source(10_000, 32, false); let original = StatisticsContext::new().compute(input.as_ref(), &StatisticsArgs::new())?; assert_eq!( - registry().compute_base(input.as_ref())?.total_byte_size, + registry_stats(input.as_ref())?.total_byte_size, original.total_byte_size ); let keys = inner_join(distinct_keys(input)?, source(5_000, 32, true))?; - let stats = registry().compute_base(keys.as_ref())?; + let stats = registry_stats(keys.as_ref())?; assert_eq!(stats.column_statistics[0].byte_size, Precision::Absent); assert_eq!(stats.total_byte_size, Precision::Absent); Ok(()) @@ -227,10 +233,7 @@ mod tests { NullEquality::NullEqualsNothing, false, )?; - assert_eq!( - registry().compute_base(&join)?.total_byte_size, - Precision::Absent - ); + assert_eq!(registry_stats(&join)?.total_byte_size, Precision::Absent); } Ok(()) } diff --git a/datafusion/distributed/src/events/route_tasks.rs b/datafusion/distributed/src/events/route_tasks.rs index fe5bd4e65..e1de7d07d 100644 --- a/datafusion/distributed/src/events/route_tasks.rs +++ b/datafusion/distributed/src/events/route_tasks.rs @@ -186,6 +186,7 @@ mod tests { task_number: 0, }, task_count: 1, + stage_plan: &plan, task_specialized_plan: &plan, dialer: &dialer, };