From f1d1660dfbf0eadc674feb5ada7eed1bde0bbee8 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:51:35 +0000 Subject: [PATCH 1/2] feat: add a way to add a filter to an Iceberg file scan task The next commit gives the Iceberg reader the scan's dynamic filters, so it can skip row groups. The reader only uses the filter stored on each FileScanTask, and iceberg-rust has no way to change that filter. and_predicate works around this. It turns the task's filter back into an unbound predicate, ANDs the new predicate onto it, binds the result and builds a copy of the task with it. Nothing calls it yet. The copy sets each of the task's fields by hand. A snapshot test lists the task's fields, so it fails if iceberg-rust adds a field the copy would leave out. --- .../src/common/dynamic_filters/mod.rs | 1 + .../src/common/dynamic_filters/task_filter.rs | 232 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 7cc259ed0..0ef97b583 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod partition_filter; pub(crate) mod physical_predicate; pub(crate) mod pruner; +pub(crate) mod task_filter; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs new file mode 100644 index 000000000..2794d59ed --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs @@ -0,0 +1,232 @@ +//! Adds a filter to a `FileScanTask`. +//! +//! iceberg-rust has no API for this. A task's filter can't be changed, and iceberg-rust has no +//! public way to AND two bound filters. So we turn the task's filter back into an unbound one, +//! AND ours onto it, bind the result, and build a new task with it. +//! +//! If iceberg-rust adds an API for this, delete this file and use the API. + +use iceberg::Result; +use iceberg::expr::{ + BinaryExpression, Bind, BoundPredicate, BoundReference, Predicate, Reference, SetExpression, + UnaryExpression, +}; +use iceberg::scan::FileScanTask; +use iceberg::spec::Schema; + +/// Returns a copy of `task` whose filter is its existing filter AND `predicate`. Returns `None` +/// if that fails. +pub(crate) fn and_predicate(task: &FileScanTask, predicate: Predicate) -> Option { + let combined = match task.predicate() { + Some(existing) => unbind(existing, task.schema())?.and(predicate), + None => predicate, + }; + let combined = combined + .bind(task.schema_ref(), task.case_sensitive()) + .ok()?; + with_predicate(task, combined).ok() +} + +/// Turns a bound predicate back into an unbound one, so we can AND it with another predicate and +/// bind it again. Each column gets its full name from `schema`, like `location.lat`. Returns +/// `None` if a column isn't in `schema`. +fn unbind(predicate: &BoundPredicate, schema: &Schema) -> Option { + let reference = |column: &BoundReference| { + schema + .name_by_field_id(column.field().id) + .map(Reference::new) + }; + Some(match predicate { + BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, + BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.and(unbind(right, schema)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.or(unbind(right, schema)?) + } + BoundPredicate::Not(expr) => { + let [inner] = expr.inputs(); + !unbind(inner, schema)? + } + BoundPredicate::Unary(expr) => { + Predicate::Unary(UnaryExpression::new(expr.op(), reference(expr.term())?)) + } + BoundPredicate::Binary(expr) => Predicate::Binary(BinaryExpression::new( + expr.op(), + reference(expr.term())?, + expr.literal().clone(), + )), + BoundPredicate::Set(expr) => Predicate::Set(SetExpression::new( + expr.op(), + reference(expr.term())?, + expr.literals().clone(), + )), + }) +} + +/// Returns a copy of `task` with `predicate` as its filter. Every other field is copied as is. +/// This is only required because iceberg-rust doesn't provide a way to add predicates to an existing FileScanTask +fn with_predicate(task: &FileScanTask, predicate: BoundPredicate) -> Result { + FileScanTask::builder() + .with_file_size_in_bytes(task.file_size_in_bytes()) + .with_start(task.start()) + .with_length(task.length()) + .with_record_count(task.record_count()) + .with_first_row_id(task.first_row_id()) + .with_data_sequence_number(task.data_sequence_number()) + .with_data_file_path(task.data_file_path().to_string()) + .with_data_file_format(task.data_file_format()) + .with_schema(task.schema_ref()) + .with_project_field_ids(task.project_field_ids().to_vec()) + .with_predicate(Some(predicate)) + .with_deletes(task.deletes().to_vec()) + .with_partition(task.partition().cloned()) + .with_partition_spec(task.partition_spec().cloned()) + .with_name_mapping(task.name_mapping().cloned()) + .with_unified_partition_type(task.unified_partition_type().cloned()) + .with_sort_order_id(task.sort_order_id()) + .with_sort_order(task.sort_order().cloned()) + .with_case_sensitive(task.case_sensitive()) + .with_key_metadata(task.key_metadata().map(Box::from)) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::scan::FileScanTaskDeleteFile; + use iceberg::spec::{ + DataContentType, DataFileFormat, Datum, Literal, MappedField, NameMapping, NestedField, + PartitionSpec, PrimitiveType, SortOrder, Struct, StructType, Transform, Type, + }; + use std::error::Error; + use std::sync::Arc; + + #[test] + fn unbinding_and_binding_again_gives_the_same_predicate() + -> std::result::Result<(), Box> { + let location = StructType::new(vec![ + NestedField::optional(3, "lat", Type::Primitive(PrimitiveType::Int)).into(), + ]); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "location", Type::Struct(location)).into(), + ]) + .build()?, + ); + let id = || Reference::new("id"); + let predicate = id() + .less_than(Datum::long(5)) + .and(!id().is_in([Datum::long(1), Datum::long(2)])) + .or(Reference::new("name").starts_with(Datum::string("a"))) + .or(Reference::new("location.lat").is_null()) + .bind(Arc::clone(&schema), true)?; + + let unbound = unbind(&predicate, &schema).ok_or("a column isn't in the schema")?; + + assert_eq!(unbound.bind(schema, true)?, predicate); + Ok(()) + } + + /// `with_predicate` copies a task one field at a time. Every field of the task below differs + /// from its default, so the `assert_eq!` fails if the copy leaves a field out. If iceberg-rust + /// adds a field, the copy would silently leave it out. The snapshot lists the task's fields, + /// so it fails when a field is added. + #[test] + fn the_copy_of_a_task_keeps_every_field() -> std::result::Result<(), Box> { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build()?, + ); + let spec = PartitionSpec::builder(Arc::clone(&schema)) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let deletes = FileScanTaskDeleteFile::builder() + .with_file_path("deletes.parquet".to_string()) + .with_file_size_in_bytes(1) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build()?; + let name_mapping = NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )]); + let task = FileScanTask::builder() + .with_file_size_in_bytes(1) + .with_start(2) + .with_length(3) + .with_record_count(Some(4)) + .with_first_row_id(Some(5)) + .with_data_sequence_number(Some(6)) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::clone(&schema)) + .with_project_field_ids(vec![1]) + .with_predicate(Some( + Reference::new("id") + .is_null() + .bind(Arc::clone(&schema), true)?, + )) + .with_deletes(vec![deletes]) + .with_partition(Some(Struct::from_iter([Some(Literal::long(7))]))) + .with_unified_partition_type(Some(Arc::new(spec.partition_type(&schema)?))) + .with_partition_spec(Some(Arc::new(spec))) + .with_name_mapping(Some(Arc::new(name_mapping))) + .with_sort_order_id(Some(8)) + .with_sort_order(Some(Arc::new(SortOrder::unsorted_order()))) + .with_case_sensitive(true) + .with_key_metadata(Some(Box::from(b"key".as_slice()))) + .build()?; + let predicate = task.predicate().ok_or("the task has no filter")?; + + let copy = with_predicate(&task, predicate.clone())?; + + assert_eq!(copy, task); + insta::assert_snapshot!(field_names(&task).join("\n"), @" + file_size_in_bytes + start + length + record_count + first_row_id + data_sequence_number + data_file_path + data_file_format + schema + project_field_ids + predicate + deletes + partition + partition_spec + name_mapping + unified_partition_type + sort_order_id + sort_order + case_sensitive + key_metadata + "); + Ok(()) + } + + /// Reads a struct's field names from its pretty-printed `Debug` output. The top-level fields + /// are the lines indented by exactly four spaces. + fn field_names(value: &impl std::fmt::Debug) -> Vec { + format!("{value:#?}") + .lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter(|line| !line.starts_with(' ')) + .filter_map(|line| line.split_once(':')) + .map(|(name, _)| name.to_string()) + .collect() + } +} From d5bec231bda6d97e4c545c64ac743b707669f1a9 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:51:44 +0000 Subject: [PATCH 2/2] feat: give the Iceberg reader dynamic filters to skip row groups With iceberg.dynamic_filtering_enabled on, the scan now gives the reader the newest dynamic filters as it starts each file. The reader uses them to skip row groups, and to drop the rows that don't match. This helps joins on columns that aren't partition columns, and top-N. The reader checks every row once per IN list value. When the IN lists hold more than 200 values in total, we leave them out and keep the min/max bounds. The reader reads a column the file doesn't store as NULL. So a filter on an identity partition column, or on a column added later with a default value, can drop every row of such a file. We accept this for now. Any error leaves the file's task as it was. --- datafusion/distributed-iceberg/README.md | 7 +- .../src/common/dynamic_filters/mod.rs | 1 + .../src/common/dynamic_filters/pruner.rs | 25 +- .../dynamic_filters/row_group_filter.rs | 101 ++++++++ .../src/common/scan_config.rs | 13 +- datafusion/distributed-iceberg/src/config.rs | 8 +- .../tests/row_group_skipping.rs | 234 ++++++++++++++++++ 7 files changed, 378 insertions(+), 11 deletions(-) create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs create mode 100644 datafusion/distributed-iceberg/tests/row_group_skipping.rs diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 9c5b6507b..5c3ddfdbd 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -96,7 +96,12 @@ return `PushedDown::No` for these filters, so DataFusion still checks every row. or aggregate only computes a dynamic filter when a scan below it is a consumer of that filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consumers of the dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as -`dynamic_filters=`. Scans don't skip data with these filters yet. +`dynamic_filters=`. Just before iceberg-rust's Parquet reader opens each file, the scan +checks the latest version of the filters: + +- It skips the file if the file's partition value can't match them. +- Otherwise, it gives the filters to the Parquet reader, which uses them to skip row groups, + and to drop the rows that don't match. ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 0ef97b583..38a122662 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod partition_filter; pub(crate) mod physical_predicate; pub(crate) mod pruner; +pub(crate) mod row_group_filter; pub(crate) mod task_filter; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs index 2f75995b8..e26bf5bc0 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs @@ -13,9 +13,12 @@ use iceberg::scan::FileScanTask; use crate::common::dynamic_filters::partition_filter::PartitionFilter; use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; +use crate::common::dynamic_filters::row_group_filter::reader_filter; +use crate::common::dynamic_filters::task_filter::and_predicate; -/// Uses the scan's dynamic filters on each file just before the reader starts it. Skips files -/// whose partition value can't match the filters. +/// Applies the scan's dynamic filters to each file just before iceberg-rust's `ArrowReader` opens +/// it. It skips files whose partition value can't match the filters. For the other files, it can +/// add the filters to the file's own filter, so `ArrowReader` can skip row groups. pub(crate) struct DynamicFilterPruner { /// The scan's dynamic filters, combined with AND. dynamic_filters: Arc, @@ -63,6 +66,24 @@ impl DynamicFilterPruner { prune } + /// Adds the dynamic filters to the task's own filter, so iceberg-rust's `ArrowReader` can use + /// them to skip row groups. Returns the task unchanged if adding them fails. + pub(crate) fn with_reader_filter(&mut self, task: FileScanTask) -> FileScanTask { + self.refresh(); + let Some(predicate) = &self.predicate else { + return task; + }; + + // Some data files may not contain identity partition columns in their rows. + // `ArrowReader` reads a column the file doesn't store as NULL, so a filter on it drops + // every row in the file. The same goes for columns added later with a default value. + // We accept this until iceberg-rust fixes it. + // + // We AND onto the task's existing filter, so `ArrowReader` still applies the query's + // WHERE clause. + and_predicate(&task, reader_filter(predicate)).unwrap_or(task) + } + /// Whether no row in the task's partition can match the filters. fn can_skip(&mut self, task: &FileScanTask) -> bool { let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs new file mode 100644 index 000000000..870852512 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs @@ -0,0 +1,101 @@ +use iceberg::expr::Predicate; + +/// The maximum number of values we give iceberg-rust's `ArrowReader` across all the IN lists in the filter. +/// +/// `ArrowReader` checks an IN list by comparing each row with every value in it, one value at a +/// time. So the more values there are, the slower every row gets. A join's filter can have +/// thousands of values. DataFusion keeps each IN list to at most 150 values, but a partitioned +/// join has one list per partition, and the filter combines them with OR. +/// +/// When the filter has more values than this, we leave out all of its IN lists. `ArrowReader` +/// still gets the join's min and max values, so it can still skip row groups outside that range. +/// We use 200 because Iceberg's own row group checks ignore any IN list longer than that. This +/// limit can be removed once `ArrowReader` checks IN lists more optimally (eg. with a hash set) +const MAX_IN_LIST_VALUES: usize = 200; + +/// Returns the dynamic filters to give iceberg-rust's `ArrowReader`. `ArrowReader` uses them to +/// skip row groups, and to drop the rows that don't match. +/// +/// `predicate` is the dynamic filters, converted to Iceberg's filter type. +/// +/// If the filters have too many IN list values, we leave out every IN list. See +/// [`MAX_IN_LIST_VALUES`]. +pub(crate) fn reader_filter(predicate: &Predicate) -> Predicate { + if count_in_list_values(predicate) <= MAX_IN_LIST_VALUES { + return predicate.clone(); + } + keep_filters(predicate, &|filter| !matches!(filter, Predicate::Set(_))) +} + +/// Walks the whole filter and adds up the values in every IN and NOT IN list it finds. +fn count_in_list_values(predicate: &Predicate) -> usize { + match predicate { + Predicate::And(expr) | Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + count_in_list_values(left) + count_in_list_values(right) + } + Predicate::Not(expr) => { + let [inner] = expr.inputs(); + count_in_list_values(inner) + } + Predicate::Set(expr) => expr.literals().len(), + _ => 0, + } +} + +/// Returns a copy of `predicate` with only the filters that `keep` accepts. We replace every +/// other filter with "always true". +/// +/// This is safe because it can only make `predicate` match more rows. So iceberg-rust's +/// `ArrowReader` may read some rows it didn't need to, but it never drops a row that matches. +/// +/// A NOT is the exception. Inside a NOT, "always true" turns into "always false", which would +/// drop rows that match. So we replace the whole NOT with "always true". Our filter converter +/// never produces a NOT, so this doesn't come up in practice. +fn keep_filters(predicate: &Predicate, keep: &dyn Fn(&Predicate) -> bool) -> Predicate { + match predicate { + Predicate::And(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).and(keep_filters(right, keep)) + } + Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).or(keep_filters(right, keep)) + } + Predicate::Not(_) => Predicate::AlwaysTrue, + filter if keep(filter) => filter.clone(), + _ => Predicate::AlwaysTrue, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::expr::Reference; + use iceberg::spec::Datum; + + #[test] + fn leaves_out_long_lists_and_keeps_their_bounds() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(id().less_than_or_equal_to(Datum::long(1000))) + .and(id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"(id >= 0) AND (id <= 1000)"); + } + + #[test] + fn leaves_out_a_not_around_a_long_list() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(!id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"id >= 0"); + } + + fn id() -> Reference { + Reference::new("id") + } +} diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 4d611c81d..6f4e2d37c 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -150,11 +150,14 @@ impl ScanConfig { // so with a LIMIT this can count files that never produced output. Skipped files // are dropped before this, and the pruner counts them. let tasks = tasks - .try_filter(move |task| { - let prune = pruner - .as_mut() - .is_some_and(|pruner| pruner.should_prune(task)); - ready(!prune) + .try_filter_map(move |mut task| { + if let Some(pruner) = pruner.as_mut() { + if pruner.should_prune(&task) { + return ready(Ok(None)); + } + task = pruner.with_reader_filter(task); + } + ready(Ok(Some(task))) }) .inspect_ok(move |_| files_opened.add(1)) .boxed(); diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 2f9e38c53..3c7f6f94e 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -18,9 +18,11 @@ extensions_options! { pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. pub eager_file_planning: bool, default = false - /// (Experimental) Whether Iceberg scans are consumers of the dynamic filters pushed - /// into them, so joins and aggregates compute those filters. Scans don't skip data - /// with them yet. + /// (Experimental) Whether Iceberg scans use the dynamic filters pushed into them. When + /// on, joins, sorts with a limit and aggregates build these filters while the query + /// runs. Scans use them to skip files whose partition value can't match. Scans also give + /// them to iceberg-rust's Parquet reader, which uses them to skip row groups and to drop + /// the rows that don't match. pub dynamic_filtering_enabled: bool, default = false } } diff --git a/datafusion/distributed-iceberg/tests/row_group_skipping.rs b/datafusion/distributed-iceberg/tests/row_group_skipping.rs new file mode 100644 index 000000000..933691b8d --- /dev/null +++ b/datafusion/distributed-iceberg/tests/row_group_skipping.rs @@ -0,0 +1,234 @@ +mod tests { + use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion_distributed::test_utils::metrics::node_metrics; + use datafusion_distributed::{ + DistributedExt, DistributedLeafExec, DistributedMetricsFormat, + rewrite_distributed_plan_with_metrics, + }; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, + }; + use iceberg::TableIdent; + use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; + use parquet::file::properties::WriterProperties; + use std::error::Error; + use std::sync::Arc; + + // The join keeps one id from each of the two `events` files. + const TWO_IDS: &str = "SELECT events.id FROM events \ + JOIN (VALUES (5), (150005)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + + #[tokio::test] + async fn the_reader_skips_row_groups_the_join_filter_rules_out() -> Result<(), Box> { + let query = "SELECT events.id, events.payload FROM events \ + JOIN (VALUES (150005), (150010)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + let with_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + let without_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", false) + .build() + .await?; + + let (with_filter_results, with_filter_plan) = run(&with_filter, query).await?; + let (without_filter_results, without_filter_plan) = run(&without_filter, query).await?; + + insta::assert_snapshot!(with_filter_results, @" + +--------+----------------------------------------+ + | id | payload | + +--------+----------------------------------------+ + | 150005 | event 150005 | + | 150010 | event 150010 | + +--------+----------------------------------------+ + "); + assert_eq!(with_filter_results, without_filter_results); + // `DataSourceExec` 0 is the join's `VALUES` table, and 1 is the `events` scan. + // Both ids are in one row group of the second file, so with the filter iceberg-rust's + // `ArrowReader` reads 1 of the 40 row groups. Both runs also read the last 512 KiB of each + // file to get its footer, so we only check that the filter cuts the bytes read by more + // than 5x. + let with_filter_bytes = + node_metrics::(&with_filter_plan, "bytes_scanned", 1); + let without_filter_bytes = + node_metrics::(&without_filter_plan, "bytes_scanned", 1); + assert!( + with_filter_bytes * 5 < without_filter_bytes, + "read {with_filter_bytes} bytes with the join filter and {without_filter_bytes} without" + ); + Ok(()) + } + + #[tokio::test] + async fn eager_planning_workers_give_the_reader_the_join_filter() -> Result<(), Box> + { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let (results, plan) = run(&harness, TWO_IDS).await?; + + insta::assert_snapshot!(results, @" + +--------+ + | id | + +--------+ + | 5 | + | 150005 | + +--------+ + "); + // The coordinator plans the files and sends them to the workers. Each worker still gives + // `ArrowReader` the join filter, so `ArrowReader` drops the rows it rules out and the scan + // returns only the two ids. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn the_top_n_filter_reaches_files_that_start_later() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so the top-N filter can tighten before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let (results, plan) = run(&harness, "SELECT id FROM events ORDER BY id LIMIT 3").await?; + + insta::assert_snapshot!(results, @" + +----+ + | id | + +----+ + | 0 | + | 1 | + | 2 | + +----+ + "); + // When the first file starts, the top 3 aren't known yet, so the scan returns all 100,000 + // of its rows. By the time the second file starts, they are, and `ArrowReader` drops + // every row in it. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 100_000 + ); + Ok(()) + } + + #[tokio::test] + async fn the_reader_keeps_the_where_filter_too() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + + let (results, plan) = run( + &harness, + "SELECT events.id, events.kind FROM events \ + JOIN (VALUES (5), (60000), (150004)) AS ids(id) ON events.id = ids.id \ + WHERE events.kind = 'click' ORDER BY events.id", + ) + .await?; + + insta::assert_snapshot!(results, @" + +--------+-------+ + | id | kind | + +--------+-------+ + | 60000 | click | + | 150004 | click | + +--------+-------+ + "); + // `ArrowReader` applies both filters. The join filter keeps ids 5, 60000 and 150004. The + // `WHERE` filter keeps only clicks, which drops the odd id 5. If adding the join filter + // lost the `WHERE` filter, the scan would return 3 rows. + assert_eq!(node_metrics::(&plan, "output_rows", 1), 2); + Ok(()) + } + + /// Runs `sql`, and returns its results and its plan with the metrics from the run. + async fn run( + harness: &IcebergTestHarness, + sql: &str, + ) -> Result<(String, Arc), Box> { + let plan = harness.physical_plan(sql).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + Ok((pretty_format_batches(&batches)?.to_string(), plan)) + } + + // The `events` table has two files of 100,000 rows each, sorted by `id`: + // - The first file has ids 0 to 99,999, and the second has 100,000 to 199,999. + // - `kind` is "click" for even ids and "view" for odd ones. + // - Each file has 20 row groups of 5,000 rows. + // - Each file is larger than the 512 KiB `ArrowReader` reads first to find the footer, so + // skipping row groups reads fewer bytes. + async fn events_by_id() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "kind", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "payload", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "events"])?, schema) + .create() + .await?; + let writer_properties = WriterProperties::builder() + .set_max_row_group_row_count(Some(5_000)) + .build(); + let mut append = table.append().writer_properties(writer_properties); + for ids in [0..100_000, 100_000..200_000] { + let kinds: Vec<&str> = ids + .clone() + .map(|id| if id % 2 == 0 { "click" } else { "view" }) + .collect(); + let payloads: Vec = ids.clone().map(|id| format!("event {id:>32}")).collect(); + let ids: ArrayRef = Arc::new(Int64Array::from_iter_values(ids)); + let kinds: ArrayRef = Arc::new(StringArray::from(kinds)); + let payloads: ArrayRef = Arc::new(StringArray::from(payloads)); + append = append.file(RecordBatch::try_from_iter([ + ("id", ids), + ("kind", kinds), + ("payload", payloads), + ])?); + } + append.commit().await?; + Ok(table) + } +}