diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 9c5b6507b..5c3ddfdbd 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -96,7 +96,12 @@ return `PushedDown::No` for these filters, so DataFusion still checks every row. or aggregate only computes a dynamic filter when a scan below it is a consumer of that filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consumers of the dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as -`dynamic_filters=`. Scans don't skip data with these filters yet. +`dynamic_filters=`. Just before iceberg-rust's Parquet reader opens each file, the scan +checks the latest version of the filters: + +- It skips the file if the file's partition value can't match them. +- Otherwise, it gives the filters to the Parquet reader, which uses them to skip row groups, + and to drop the rows that don't match. ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 7cc259ed0..38a122662 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod partition_filter; pub(crate) mod physical_predicate; pub(crate) mod pruner; +pub(crate) mod row_group_filter; +pub(crate) mod task_filter; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs index 2f75995b8..e26bf5bc0 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs @@ -13,9 +13,12 @@ use iceberg::scan::FileScanTask; use crate::common::dynamic_filters::partition_filter::PartitionFilter; use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; +use crate::common::dynamic_filters::row_group_filter::reader_filter; +use crate::common::dynamic_filters::task_filter::and_predicate; -/// Uses the scan's dynamic filters on each file just before the reader starts it. Skips files -/// whose partition value can't match the filters. +/// Applies the scan's dynamic filters to each file just before iceberg-rust's `ArrowReader` opens +/// it. It skips files whose partition value can't match the filters. For the other files, it can +/// add the filters to the file's own filter, so `ArrowReader` can skip row groups. pub(crate) struct DynamicFilterPruner { /// The scan's dynamic filters, combined with AND. dynamic_filters: Arc, @@ -63,6 +66,24 @@ impl DynamicFilterPruner { prune } + /// Adds the dynamic filters to the task's own filter, so iceberg-rust's `ArrowReader` can use + /// them to skip row groups. Returns the task unchanged if adding them fails. + pub(crate) fn with_reader_filter(&mut self, task: FileScanTask) -> FileScanTask { + self.refresh(); + let Some(predicate) = &self.predicate else { + return task; + }; + + // Some data files may not contain identity partition columns in their rows. + // `ArrowReader` reads a column the file doesn't store as NULL, so a filter on it drops + // every row in the file. The same goes for columns added later with a default value. + // We accept this until iceberg-rust fixes it. + // + // We AND onto the task's existing filter, so `ArrowReader` still applies the query's + // WHERE clause. + and_predicate(&task, reader_filter(predicate)).unwrap_or(task) + } + /// Whether no row in the task's partition can match the filters. fn can_skip(&mut self, task: &FileScanTask) -> bool { let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs new file mode 100644 index 000000000..870852512 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs @@ -0,0 +1,101 @@ +use iceberg::expr::Predicate; + +/// The maximum number of values we give iceberg-rust's `ArrowReader` across all the IN lists in the filter. +/// +/// `ArrowReader` checks an IN list by comparing each row with every value in it, one value at a +/// time. So the more values there are, the slower every row gets. A join's filter can have +/// thousands of values. DataFusion keeps each IN list to at most 150 values, but a partitioned +/// join has one list per partition, and the filter combines them with OR. +/// +/// When the filter has more values than this, we leave out all of its IN lists. `ArrowReader` +/// still gets the join's min and max values, so it can still skip row groups outside that range. +/// We use 200 because Iceberg's own row group checks ignore any IN list longer than that. This +/// limit can be removed once `ArrowReader` checks IN lists more optimally (eg. with a hash set) +const MAX_IN_LIST_VALUES: usize = 200; + +/// Returns the dynamic filters to give iceberg-rust's `ArrowReader`. `ArrowReader` uses them to +/// skip row groups, and to drop the rows that don't match. +/// +/// `predicate` is the dynamic filters, converted to Iceberg's filter type. +/// +/// If the filters have too many IN list values, we leave out every IN list. See +/// [`MAX_IN_LIST_VALUES`]. +pub(crate) fn reader_filter(predicate: &Predicate) -> Predicate { + if count_in_list_values(predicate) <= MAX_IN_LIST_VALUES { + return predicate.clone(); + } + keep_filters(predicate, &|filter| !matches!(filter, Predicate::Set(_))) +} + +/// Walks the whole filter and adds up the values in every IN and NOT IN list it finds. +fn count_in_list_values(predicate: &Predicate) -> usize { + match predicate { + Predicate::And(expr) | Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + count_in_list_values(left) + count_in_list_values(right) + } + Predicate::Not(expr) => { + let [inner] = expr.inputs(); + count_in_list_values(inner) + } + Predicate::Set(expr) => expr.literals().len(), + _ => 0, + } +} + +/// Returns a copy of `predicate` with only the filters that `keep` accepts. We replace every +/// other filter with "always true". +/// +/// This is safe because it can only make `predicate` match more rows. So iceberg-rust's +/// `ArrowReader` may read some rows it didn't need to, but it never drops a row that matches. +/// +/// A NOT is the exception. Inside a NOT, "always true" turns into "always false", which would +/// drop rows that match. So we replace the whole NOT with "always true". Our filter converter +/// never produces a NOT, so this doesn't come up in practice. +fn keep_filters(predicate: &Predicate, keep: &dyn Fn(&Predicate) -> bool) -> Predicate { + match predicate { + Predicate::And(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).and(keep_filters(right, keep)) + } + Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).or(keep_filters(right, keep)) + } + Predicate::Not(_) => Predicate::AlwaysTrue, + filter if keep(filter) => filter.clone(), + _ => Predicate::AlwaysTrue, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::expr::Reference; + use iceberg::spec::Datum; + + #[test] + fn leaves_out_long_lists_and_keeps_their_bounds() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(id().less_than_or_equal_to(Datum::long(1000))) + .and(id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"(id >= 0) AND (id <= 1000)"); + } + + #[test] + fn leaves_out_a_not_around_a_long_list() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(!id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"id >= 0"); + } + + fn id() -> Reference { + Reference::new("id") + } +} diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs new file mode 100644 index 000000000..2794d59ed --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs @@ -0,0 +1,232 @@ +//! Adds a filter to a `FileScanTask`. +//! +//! iceberg-rust has no API for this. A task's filter can't be changed, and iceberg-rust has no +//! public way to AND two bound filters. So we turn the task's filter back into an unbound one, +//! AND ours onto it, bind the result, and build a new task with it. +//! +//! If iceberg-rust adds an API for this, delete this file and use the API. + +use iceberg::Result; +use iceberg::expr::{ + BinaryExpression, Bind, BoundPredicate, BoundReference, Predicate, Reference, SetExpression, + UnaryExpression, +}; +use iceberg::scan::FileScanTask; +use iceberg::spec::Schema; + +/// Returns a copy of `task` whose filter is its existing filter AND `predicate`. Returns `None` +/// if that fails. +pub(crate) fn and_predicate(task: &FileScanTask, predicate: Predicate) -> Option { + let combined = match task.predicate() { + Some(existing) => unbind(existing, task.schema())?.and(predicate), + None => predicate, + }; + let combined = combined + .bind(task.schema_ref(), task.case_sensitive()) + .ok()?; + with_predicate(task, combined).ok() +} + +/// Turns a bound predicate back into an unbound one, so we can AND it with another predicate and +/// bind it again. Each column gets its full name from `schema`, like `location.lat`. Returns +/// `None` if a column isn't in `schema`. +fn unbind(predicate: &BoundPredicate, schema: &Schema) -> Option { + let reference = |column: &BoundReference| { + schema + .name_by_field_id(column.field().id) + .map(Reference::new) + }; + Some(match predicate { + BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, + BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.and(unbind(right, schema)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.or(unbind(right, schema)?) + } + BoundPredicate::Not(expr) => { + let [inner] = expr.inputs(); + !unbind(inner, schema)? + } + BoundPredicate::Unary(expr) => { + Predicate::Unary(UnaryExpression::new(expr.op(), reference(expr.term())?)) + } + BoundPredicate::Binary(expr) => Predicate::Binary(BinaryExpression::new( + expr.op(), + reference(expr.term())?, + expr.literal().clone(), + )), + BoundPredicate::Set(expr) => Predicate::Set(SetExpression::new( + expr.op(), + reference(expr.term())?, + expr.literals().clone(), + )), + }) +} + +/// Returns a copy of `task` with `predicate` as its filter. Every other field is copied as is. +/// This is only required because iceberg-rust doesn't provide a way to add predicates to an existing FileScanTask +fn with_predicate(task: &FileScanTask, predicate: BoundPredicate) -> Result { + FileScanTask::builder() + .with_file_size_in_bytes(task.file_size_in_bytes()) + .with_start(task.start()) + .with_length(task.length()) + .with_record_count(task.record_count()) + .with_first_row_id(task.first_row_id()) + .with_data_sequence_number(task.data_sequence_number()) + .with_data_file_path(task.data_file_path().to_string()) + .with_data_file_format(task.data_file_format()) + .with_schema(task.schema_ref()) + .with_project_field_ids(task.project_field_ids().to_vec()) + .with_predicate(Some(predicate)) + .with_deletes(task.deletes().to_vec()) + .with_partition(task.partition().cloned()) + .with_partition_spec(task.partition_spec().cloned()) + .with_name_mapping(task.name_mapping().cloned()) + .with_unified_partition_type(task.unified_partition_type().cloned()) + .with_sort_order_id(task.sort_order_id()) + .with_sort_order(task.sort_order().cloned()) + .with_case_sensitive(task.case_sensitive()) + .with_key_metadata(task.key_metadata().map(Box::from)) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::scan::FileScanTaskDeleteFile; + use iceberg::spec::{ + DataContentType, DataFileFormat, Datum, Literal, MappedField, NameMapping, NestedField, + PartitionSpec, PrimitiveType, SortOrder, Struct, StructType, Transform, Type, + }; + use std::error::Error; + use std::sync::Arc; + + #[test] + fn unbinding_and_binding_again_gives_the_same_predicate() + -> std::result::Result<(), Box> { + let location = StructType::new(vec![ + NestedField::optional(3, "lat", Type::Primitive(PrimitiveType::Int)).into(), + ]); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "location", Type::Struct(location)).into(), + ]) + .build()?, + ); + let id = || Reference::new("id"); + let predicate = id() + .less_than(Datum::long(5)) + .and(!id().is_in([Datum::long(1), Datum::long(2)])) + .or(Reference::new("name").starts_with(Datum::string("a"))) + .or(Reference::new("location.lat").is_null()) + .bind(Arc::clone(&schema), true)?; + + let unbound = unbind(&predicate, &schema).ok_or("a column isn't in the schema")?; + + assert_eq!(unbound.bind(schema, true)?, predicate); + Ok(()) + } + + /// `with_predicate` copies a task one field at a time. Every field of the task below differs + /// from its default, so the `assert_eq!` fails if the copy leaves a field out. If iceberg-rust + /// adds a field, the copy would silently leave it out. The snapshot lists the task's fields, + /// so it fails when a field is added. + #[test] + fn the_copy_of_a_task_keeps_every_field() -> std::result::Result<(), Box> { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build()?, + ); + let spec = PartitionSpec::builder(Arc::clone(&schema)) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let deletes = FileScanTaskDeleteFile::builder() + .with_file_path("deletes.parquet".to_string()) + .with_file_size_in_bytes(1) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build()?; + let name_mapping = NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )]); + let task = FileScanTask::builder() + .with_file_size_in_bytes(1) + .with_start(2) + .with_length(3) + .with_record_count(Some(4)) + .with_first_row_id(Some(5)) + .with_data_sequence_number(Some(6)) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::clone(&schema)) + .with_project_field_ids(vec![1]) + .with_predicate(Some( + Reference::new("id") + .is_null() + .bind(Arc::clone(&schema), true)?, + )) + .with_deletes(vec![deletes]) + .with_partition(Some(Struct::from_iter([Some(Literal::long(7))]))) + .with_unified_partition_type(Some(Arc::new(spec.partition_type(&schema)?))) + .with_partition_spec(Some(Arc::new(spec))) + .with_name_mapping(Some(Arc::new(name_mapping))) + .with_sort_order_id(Some(8)) + .with_sort_order(Some(Arc::new(SortOrder::unsorted_order()))) + .with_case_sensitive(true) + .with_key_metadata(Some(Box::from(b"key".as_slice()))) + .build()?; + let predicate = task.predicate().ok_or("the task has no filter")?; + + let copy = with_predicate(&task, predicate.clone())?; + + assert_eq!(copy, task); + insta::assert_snapshot!(field_names(&task).join("\n"), @" + file_size_in_bytes + start + length + record_count + first_row_id + data_sequence_number + data_file_path + data_file_format + schema + project_field_ids + predicate + deletes + partition + partition_spec + name_mapping + unified_partition_type + sort_order_id + sort_order + case_sensitive + key_metadata + "); + Ok(()) + } + + /// Reads a struct's field names from its pretty-printed `Debug` output. The top-level fields + /// are the lines indented by exactly four spaces. + fn field_names(value: &impl std::fmt::Debug) -> Vec { + format!("{value:#?}") + .lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter(|line| !line.starts_with(' ')) + .filter_map(|line| line.split_once(':')) + .map(|(name, _)| name.to_string()) + .collect() + } +} diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 4d611c81d..6f4e2d37c 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -150,11 +150,14 @@ impl ScanConfig { // so with a LIMIT this can count files that never produced output. Skipped files // are dropped before this, and the pruner counts them. let tasks = tasks - .try_filter(move |task| { - let prune = pruner - .as_mut() - .is_some_and(|pruner| pruner.should_prune(task)); - ready(!prune) + .try_filter_map(move |mut task| { + if let Some(pruner) = pruner.as_mut() { + if pruner.should_prune(&task) { + return ready(Ok(None)); + } + task = pruner.with_reader_filter(task); + } + ready(Ok(Some(task))) }) .inspect_ok(move |_| files_opened.add(1)) .boxed(); diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 2f9e38c53..3c7f6f94e 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -18,9 +18,11 @@ extensions_options! { pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. pub eager_file_planning: bool, default = false - /// (Experimental) Whether Iceberg scans are consumers of the dynamic filters pushed - /// into them, so joins and aggregates compute those filters. Scans don't skip data - /// with them yet. + /// (Experimental) Whether Iceberg scans use the dynamic filters pushed into them. When + /// on, joins, sorts with a limit and aggregates build these filters while the query + /// runs. Scans use them to skip files whose partition value can't match. Scans also give + /// them to iceberg-rust's Parquet reader, which uses them to skip row groups and to drop + /// the rows that don't match. pub dynamic_filtering_enabled: bool, default = false } } diff --git a/datafusion/distributed-iceberg/tests/row_group_skipping.rs b/datafusion/distributed-iceberg/tests/row_group_skipping.rs new file mode 100644 index 000000000..933691b8d --- /dev/null +++ b/datafusion/distributed-iceberg/tests/row_group_skipping.rs @@ -0,0 +1,234 @@ +mod tests { + use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion_distributed::test_utils::metrics::node_metrics; + use datafusion_distributed::{ + DistributedExt, DistributedLeafExec, DistributedMetricsFormat, + rewrite_distributed_plan_with_metrics, + }; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, + }; + use iceberg::TableIdent; + use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; + use parquet::file::properties::WriterProperties; + use std::error::Error; + use std::sync::Arc; + + // The join keeps one id from each of the two `events` files. + const TWO_IDS: &str = "SELECT events.id FROM events \ + JOIN (VALUES (5), (150005)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + + #[tokio::test] + async fn the_reader_skips_row_groups_the_join_filter_rules_out() -> Result<(), Box> { + let query = "SELECT events.id, events.payload FROM events \ + JOIN (VALUES (150005), (150010)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + let with_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + let without_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", false) + .build() + .await?; + + let (with_filter_results, with_filter_plan) = run(&with_filter, query).await?; + let (without_filter_results, without_filter_plan) = run(&without_filter, query).await?; + + insta::assert_snapshot!(with_filter_results, @" + +--------+----------------------------------------+ + | id | payload | + +--------+----------------------------------------+ + | 150005 | event 150005 | + | 150010 | event 150010 | + +--------+----------------------------------------+ + "); + assert_eq!(with_filter_results, without_filter_results); + // `DataSourceExec` 0 is the join's `VALUES` table, and 1 is the `events` scan. + // Both ids are in one row group of the second file, so with the filter iceberg-rust's + // `ArrowReader` reads 1 of the 40 row groups. Both runs also read the last 512 KiB of each + // file to get its footer, so we only check that the filter cuts the bytes read by more + // than 5x. + let with_filter_bytes = + node_metrics::(&with_filter_plan, "bytes_scanned", 1); + let without_filter_bytes = + node_metrics::(&without_filter_plan, "bytes_scanned", 1); + assert!( + with_filter_bytes * 5 < without_filter_bytes, + "read {with_filter_bytes} bytes with the join filter and {without_filter_bytes} without" + ); + Ok(()) + } + + #[tokio::test] + async fn eager_planning_workers_give_the_reader_the_join_filter() -> Result<(), Box> + { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let (results, plan) = run(&harness, TWO_IDS).await?; + + insta::assert_snapshot!(results, @" + +--------+ + | id | + +--------+ + | 5 | + | 150005 | + +--------+ + "); + // The coordinator plans the files and sends them to the workers. Each worker still gives + // `ArrowReader` the join filter, so `ArrowReader` drops the rows it rules out and the scan + // returns only the two ids. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn the_top_n_filter_reaches_files_that_start_later() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so the top-N filter can tighten before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let (results, plan) = run(&harness, "SELECT id FROM events ORDER BY id LIMIT 3").await?; + + insta::assert_snapshot!(results, @" + +----+ + | id | + +----+ + | 0 | + | 1 | + | 2 | + +----+ + "); + // When the first file starts, the top 3 aren't known yet, so the scan returns all 100,000 + // of its rows. By the time the second file starts, they are, and `ArrowReader` drops + // every row in it. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 100_000 + ); + Ok(()) + } + + #[tokio::test] + async fn the_reader_keeps_the_where_filter_too() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + + let (results, plan) = run( + &harness, + "SELECT events.id, events.kind FROM events \ + JOIN (VALUES (5), (60000), (150004)) AS ids(id) ON events.id = ids.id \ + WHERE events.kind = 'click' ORDER BY events.id", + ) + .await?; + + insta::assert_snapshot!(results, @" + +--------+-------+ + | id | kind | + +--------+-------+ + | 60000 | click | + | 150004 | click | + +--------+-------+ + "); + // `ArrowReader` applies both filters. The join filter keeps ids 5, 60000 and 150004. The + // `WHERE` filter keeps only clicks, which drops the odd id 5. If adding the join filter + // lost the `WHERE` filter, the scan would return 3 rows. + assert_eq!(node_metrics::(&plan, "output_rows", 1), 2); + Ok(()) + } + + /// Runs `sql`, and returns its results and its plan with the metrics from the run. + async fn run( + harness: &IcebergTestHarness, + sql: &str, + ) -> Result<(String, Arc), Box> { + let plan = harness.physical_plan(sql).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + Ok((pretty_format_batches(&batches)?.to_string(), plan)) + } + + // The `events` table has two files of 100,000 rows each, sorted by `id`: + // - The first file has ids 0 to 99,999, and the second has 100,000 to 199,999. + // - `kind` is "click" for even ids and "view" for odd ones. + // - Each file has 20 row groups of 5,000 rows. + // - Each file is larger than the 512 KiB `ArrowReader` reads first to find the footer, so + // skipping row groups reads fewer bytes. + async fn events_by_id() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "kind", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "payload", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "events"])?, schema) + .create() + .await?; + let writer_properties = WriterProperties::builder() + .set_max_row_group_row_count(Some(5_000)) + .build(); + let mut append = table.append().writer_properties(writer_properties); + for ids in [0..100_000, 100_000..200_000] { + let kinds: Vec<&str> = ids + .clone() + .map(|id| if id % 2 == 0 { "click" } else { "view" }) + .collect(); + let payloads: Vec = ids.clone().map(|id| format!("event {id:>32}")).collect(); + let ids: ArrayRef = Arc::new(Int64Array::from_iter_values(ids)); + let kinds: ArrayRef = Arc::new(StringArray::from(kinds)); + let payloads: ArrayRef = Arc::new(StringArray::from(payloads)); + append = append.file(RecordBatch::try_from_iter([ + ("id", ids), + ("kind", kinds), + ("payload", payloads), + ])?); + } + append.commit().await?; + Ok(table) + } +}