Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion datafusion/distributed-iceberg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,12 @@ return `PushedDown::No` for these filters, so DataFusion still checks every row.
or aggregate only computes a dynamic filter when a scan below it is a consumer of that
filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consumers of the
dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as
`dynamic_filters=`. Scans don't skip data with these filters yet.
`dynamic_filters=`. Just before iceberg-rust's Parquet reader opens each file, the scan
checks the latest version of the filters:

- It skips the file if the file's partition value can't match them.
- Otherwise, it gives the filters to the Parquet reader, which uses them to skip row groups,
and to drop the rows that don't match.

```bash
cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
pub(crate) mod partition_filter;
pub(crate) mod physical_predicate;
pub(crate) mod pruner;
pub(crate) mod row_group_filter;
pub(crate) mod task_filter;
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@ use iceberg::scan::FileScanTask;

use crate::common::dynamic_filters::partition_filter::PartitionFilter;
use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate;
use crate::common::dynamic_filters::row_group_filter::reader_filter;
use crate::common::dynamic_filters::task_filter::and_predicate;

/// Uses the scan's dynamic filters on each file just before the reader starts it. Skips files
/// whose partition value can't match the filters.
/// Applies the scan's dynamic filters to each file just before iceberg-rust's `ArrowReader` opens
/// it. It skips files whose partition value can't match the filters. For the other files, it can
/// add the filters to the file's own filter, so `ArrowReader` can skip row groups.
pub(crate) struct DynamicFilterPruner {
/// The scan's dynamic filters, combined with AND.
dynamic_filters: Arc<dyn PhysicalExpr>,
Expand Down Expand Up @@ -63,6 +66,24 @@ impl DynamicFilterPruner {
prune
}

/// Adds the dynamic filters to the task's own filter, so iceberg-rust's `ArrowReader` can use
/// them to skip row groups. Returns the task unchanged if adding them fails.
pub(crate) fn with_reader_filter(&mut self, task: FileScanTask) -> FileScanTask {
self.refresh();
let Some(predicate) = &self.predicate else {
return task;
};

// Some data files may not contain identity partition columns in their rows.
// `ArrowReader` reads a column the file doesn't store as NULL, so a filter on it drops
// every row in the file. The same goes for columns added later with a default value.
// We accept this until iceberg-rust fixes it.
//
// We AND onto the task's existing filter, so `ArrowReader` still applies the query's
// WHERE clause.
and_predicate(&task, reader_filter(predicate)).unwrap_or(task)
}

/// Whether no row in the task's partition can match the filters.
fn can_skip(&mut self, task: &FileScanTask) -> bool {
let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
use iceberg::expr::Predicate;

/// The maximum number of values we give iceberg-rust's `ArrowReader` across all the IN lists in the filter.
///
/// `ArrowReader` checks an IN list by comparing each row with every value in it, one value at a
/// time. So the more values there are, the slower every row gets. A join's filter can have
/// thousands of values. DataFusion keeps each IN list to at most 150 values, but a partitioned
/// join has one list per partition, and the filter combines them with OR.
///
/// When the filter has more values than this, we leave out all of its IN lists. `ArrowReader`
/// still gets the join's min and max values, so it can still skip row groups outside that range.
/// We use 200 because Iceberg's own row group checks ignore any IN list longer than that. This
/// limit can be removed once `ArrowReader` checks IN lists more optimally (eg. with a hash set)
const MAX_IN_LIST_VALUES: usize = 200;

/// Returns the dynamic filters to give iceberg-rust's `ArrowReader`. `ArrowReader` uses them to
/// skip row groups, and to drop the rows that don't match.
///
/// `predicate` is the dynamic filters, converted to Iceberg's filter type.
///
/// If the filters have too many IN list values, we leave out every IN list. See
/// [`MAX_IN_LIST_VALUES`].
pub(crate) fn reader_filter(predicate: &Predicate) -> Predicate {
if count_in_list_values(predicate) <= MAX_IN_LIST_VALUES {
return predicate.clone();
}
keep_filters(predicate, &|filter| !matches!(filter, Predicate::Set(_)))
}

/// Walks the whole filter and adds up the values in every IN and NOT IN list it finds.
fn count_in_list_values(predicate: &Predicate) -> usize {
match predicate {
Predicate::And(expr) | Predicate::Or(expr) => {
let [left, right] = expr.inputs();
count_in_list_values(left) + count_in_list_values(right)
}
Predicate::Not(expr) => {
let [inner] = expr.inputs();
count_in_list_values(inner)
}
Predicate::Set(expr) => expr.literals().len(),
_ => 0,
}
}

/// Returns a copy of `predicate` with only the filters that `keep` accepts. We replace every
/// other filter with "always true".
///
/// This is safe because it can only make `predicate` match more rows. So iceberg-rust's
/// `ArrowReader` may read some rows it didn't need to, but it never drops a row that matches.
///
/// A NOT is the exception. Inside a NOT, "always true" turns into "always false", which would
/// drop rows that match. So we replace the whole NOT with "always true". Our filter converter
/// never produces a NOT, so this doesn't come up in practice.
fn keep_filters(predicate: &Predicate, keep: &dyn Fn(&Predicate) -> bool) -> Predicate {
match predicate {
Predicate::And(expr) => {
let [left, right] = expr.inputs();
keep_filters(left, keep).and(keep_filters(right, keep))
}
Predicate::Or(expr) => {
let [left, right] = expr.inputs();
keep_filters(left, keep).or(keep_filters(right, keep))
}
Predicate::Not(_) => Predicate::AlwaysTrue,
filter if keep(filter) => filter.clone(),
_ => Predicate::AlwaysTrue,
}
}

#[cfg(test)]
mod tests {
use super::*;
use iceberg::expr::Reference;
use iceberg::spec::Datum;

#[test]
fn leaves_out_long_lists_and_keeps_their_bounds() {
let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long);
let predicate = id()
.greater_than_or_equal_to(Datum::long(0))
.and(id().less_than_or_equal_to(Datum::long(1000)))
.and(id().is_in(values));

insta::assert_snapshot!(reader_filter(&predicate), @"(id >= 0) AND (id <= 1000)");
}

#[test]
fn leaves_out_a_not_around_a_long_list() {
let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long);
let predicate = id()
.greater_than_or_equal_to(Datum::long(0))
.and(!id().is_in(values));

insta::assert_snapshot!(reader_filter(&predicate), @"id >= 0");
}

fn id() -> Reference {
Reference::new("id")
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
//! Adds a filter to a `FileScanTask`.
//!
//! iceberg-rust has no API for this. A task's filter can't be changed, and iceberg-rust has no
//! public way to AND two bound filters. So we turn the task's filter back into an unbound one,
//! AND ours onto it, bind the result, and build a new task with it.
//!
//! If iceberg-rust adds an API for this, delete this file and use the API.

use iceberg::Result;
use iceberg::expr::{
BinaryExpression, Bind, BoundPredicate, BoundReference, Predicate, Reference, SetExpression,
UnaryExpression,
};
use iceberg::scan::FileScanTask;
use iceberg::spec::Schema;

/// Returns a copy of `task` whose filter is its existing filter AND `predicate`. Returns `None`
/// if that fails.
pub(crate) fn and_predicate(task: &FileScanTask, predicate: Predicate) -> Option<FileScanTask> {
let combined = match task.predicate() {
Some(existing) => unbind(existing, task.schema())?.and(predicate),
None => predicate,
};
let combined = combined
.bind(task.schema_ref(), task.case_sensitive())
.ok()?;
with_predicate(task, combined).ok()
}

/// Turns a bound predicate back into an unbound one, so we can AND it with another predicate and
/// bind it again. Each column gets its full name from `schema`, like `location.lat`. Returns
/// `None` if a column isn't in `schema`.
fn unbind(predicate: &BoundPredicate, schema: &Schema) -> Option<Predicate> {
let reference = |column: &BoundReference| {
schema
.name_by_field_id(column.field().id)
.map(Reference::new)
};
Some(match predicate {
BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue,
BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse,
BoundPredicate::And(expr) => {
let [left, right] = expr.inputs();
unbind(left, schema)?.and(unbind(right, schema)?)
}
BoundPredicate::Or(expr) => {
let [left, right] = expr.inputs();
unbind(left, schema)?.or(unbind(right, schema)?)
}
BoundPredicate::Not(expr) => {
let [inner] = expr.inputs();
!unbind(inner, schema)?
}
BoundPredicate::Unary(expr) => {
Predicate::Unary(UnaryExpression::new(expr.op(), reference(expr.term())?))
}
BoundPredicate::Binary(expr) => Predicate::Binary(BinaryExpression::new(
expr.op(),
reference(expr.term())?,
expr.literal().clone(),
)),
BoundPredicate::Set(expr) => Predicate::Set(SetExpression::new(
expr.op(),
reference(expr.term())?,
expr.literals().clone(),
)),
})
}

/// Returns a copy of `task` with `predicate` as its filter. Every other field is copied as is.
/// This is only required because iceberg-rust doesn't provide a way to add predicates to an existing FileScanTask
fn with_predicate(task: &FileScanTask, predicate: BoundPredicate) -> Result<FileScanTask> {
FileScanTask::builder()
.with_file_size_in_bytes(task.file_size_in_bytes())
.with_start(task.start())
.with_length(task.length())
.with_record_count(task.record_count())
.with_first_row_id(task.first_row_id())
.with_data_sequence_number(task.data_sequence_number())
.with_data_file_path(task.data_file_path().to_string())
.with_data_file_format(task.data_file_format())
.with_schema(task.schema_ref())
.with_project_field_ids(task.project_field_ids().to_vec())
.with_predicate(Some(predicate))
.with_deletes(task.deletes().to_vec())
.with_partition(task.partition().cloned())
.with_partition_spec(task.partition_spec().cloned())
.with_name_mapping(task.name_mapping().cloned())
.with_unified_partition_type(task.unified_partition_type().cloned())
.with_sort_order_id(task.sort_order_id())
.with_sort_order(task.sort_order().cloned())
.with_case_sensitive(task.case_sensitive())
.with_key_metadata(task.key_metadata().map(Box::from))
.build()
}

#[cfg(test)]
mod tests {
use super::*;
use iceberg::scan::FileScanTaskDeleteFile;
use iceberg::spec::{
DataContentType, DataFileFormat, Datum, Literal, MappedField, NameMapping, NestedField,
PartitionSpec, PrimitiveType, SortOrder, Struct, StructType, Transform, Type,
};
use std::error::Error;
use std::sync::Arc;

#[test]
fn unbinding_and_binding_again_gives_the_same_predicate()
-> std::result::Result<(), Box<dyn Error>> {
let location = StructType::new(vec![
NestedField::optional(3, "lat", Type::Primitive(PrimitiveType::Int)).into(),
]);
let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(),
NestedField::optional(4, "location", Type::Struct(location)).into(),
])
.build()?,
);
let id = || Reference::new("id");
let predicate = id()
.less_than(Datum::long(5))
.and(!id().is_in([Datum::long(1), Datum::long(2)]))
.or(Reference::new("name").starts_with(Datum::string("a")))
.or(Reference::new("location.lat").is_null())
.bind(Arc::clone(&schema), true)?;

let unbound = unbind(&predicate, &schema).ok_or("a column isn't in the schema")?;

assert_eq!(unbound.bind(schema, true)?, predicate);
Ok(())
}

/// `with_predicate` copies a task one field at a time. Every field of the task below differs
/// from its default, so the `assert_eq!` fails if the copy leaves a field out. If iceberg-rust
/// adds a field, the copy would silently leave it out. The snapshot lists the task's fields,
/// so it fails when a field is added.
#[test]
fn the_copy_of_a_task_keeps_every_field() -> std::result::Result<(), Box<dyn Error>> {
let schema = Arc::new(
Schema::builder()
.with_fields(vec![
NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
])
.build()?,
);
let spec = PartitionSpec::builder(Arc::clone(&schema))
.add_partition_field("id", "id", Transform::Identity)?
.build()?;
let deletes = FileScanTaskDeleteFile::builder()
.with_file_path("deletes.parquet".to_string())
.with_file_size_in_bytes(1)
.with_file_type(DataContentType::PositionDeletes)
.with_file_format(DataFileFormat::Parquet)
.with_partition_spec_id(0)
.build()?;
let name_mapping = NameMapping::new(vec![MappedField::new(
Some(1),
vec!["id".to_string()],
vec![],
)]);
let task = FileScanTask::builder()
.with_file_size_in_bytes(1)
.with_start(2)
.with_length(3)
.with_record_count(Some(4))
.with_first_row_id(Some(5))
.with_data_sequence_number(Some(6))
.with_data_file_path("data.parquet".to_string())
.with_data_file_format(DataFileFormat::Parquet)
.with_schema(Arc::clone(&schema))
.with_project_field_ids(vec![1])
.with_predicate(Some(
Reference::new("id")
.is_null()
.bind(Arc::clone(&schema), true)?,
))
.with_deletes(vec![deletes])
.with_partition(Some(Struct::from_iter([Some(Literal::long(7))])))
.with_unified_partition_type(Some(Arc::new(spec.partition_type(&schema)?)))
.with_partition_spec(Some(Arc::new(spec)))
.with_name_mapping(Some(Arc::new(name_mapping)))
.with_sort_order_id(Some(8))
.with_sort_order(Some(Arc::new(SortOrder::unsorted_order())))
.with_case_sensitive(true)
.with_key_metadata(Some(Box::from(b"key".as_slice())))
.build()?;
let predicate = task.predicate().ok_or("the task has no filter")?;

let copy = with_predicate(&task, predicate.clone())?;

assert_eq!(copy, task);
insta::assert_snapshot!(field_names(&task).join("\n"), @"
file_size_in_bytes
start
length
record_count
first_row_id
data_sequence_number
data_file_path
data_file_format
schema
project_field_ids
predicate
deletes
partition
partition_spec
name_mapping
unified_partition_type
sort_order_id
sort_order
case_sensitive
key_metadata
");
Ok(())
}

/// Reads a struct's field names from its pretty-printed `Debug` output. The top-level fields
/// are the lines indented by exactly four spaces.
fn field_names(value: &impl std::fmt::Debug) -> Vec<String> {
format!("{value:#?}")
.lines()
.filter_map(|line| line.strip_prefix(" "))
.filter(|line| !line.starts_with(' '))
.filter_map(|line| line.split_once(':'))
.map(|(name, _)| name.to_string())
.collect()
}
}
Loading
Loading