Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 184 additions & 39 deletions Cargo.lock

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,11 @@ datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "16
[workspace.lints.clippy]
disallowed_types = "deny"
disallowed-methods = "deny"

# datafusion-iceberg pins apache/iceberg-rust at an older Arrow 60 revision. Resolve it to the
# same revision as this workspace (apache/iceberg-rust 13f84f9: that revision plus FileScanTask
# column statistics, #3424) so FileScanTask is one type. Drop this once datafusion-iceberg
# depends on a revision with both.
[patch."https://github.com/apache/iceberg-rust.git"]
iceberg = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" }
iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" }
2 changes: 1 addition & 1 deletion datafusion/distributed-iceberg-test-tools/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ datafusion = { workspace = true, features = ["parquet", "sql"] }
datafusion-distributed = { path = "../distributed", features = ["integration"] }
datafusion-proto = { workspace = true }
# Match datafusion-distributed-iceberg's iceberg-rust revision so tables share one type.
iceberg = { git = "https://github.com/apache/iceberg-rust.git", rev = "0d5eb123495f765ea151e78c8964919d554d4b98" }
iceberg = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" }
rand = { version = "0.10", default-features = false, features = ["alloc"] }
rand_pcg = "0.10"
tempfile = "3.18"
Expand Down
13 changes: 9 additions & 4 deletions datafusion/distributed-iceberg-test-tools/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ use datafusion::common::exec_datafusion_err;
use datafusion::error::Result;
use iceberg::arrow::schema_to_arrow_schema;
use iceberg::memory::MemoryCatalog;
use iceberg::spec::{Schema, TableMetadata, Transform, UnboundPartitionSpec};
use iceberg::spec::{
Schema, TableMetadata, Transform, UnboundPartitionField, UnboundPartitionSpec,
};
use iceberg::table::Table;
use iceberg::{Catalog, MetadataLocation, TableCreation, TableIdent};
use tempfile::TempDir;
Expand Down Expand Up @@ -70,9 +72,12 @@ impl TableBuilder {
})?;
partition_spec = partition_spec
.add_partition_field(
source.id,
partition_field_name(&column, transform),
transform,
UnboundPartitionField::builder()
.source_ids(vec![source.id])
.name(partition_field_name(&column, transform))
.transform(transform)
.build()
.map_err(df_err)?,
)
.map_err(df_err)?;
}
Expand Down
8 changes: 5 additions & 3 deletions datafusion/distributed-iceberg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ async-trait = "0.1"
bytes = "1"
datafusion = { workspace = true, features = ["sql"] }
datafusion-proto = { workspace = true }
# Match datafusion-iceberg's iceberg-rust revision so FileScanTask has one shared type.
# apache/iceberg-rust 13f84f9: Arrow 60 plus FileScanTask column statistics (#3424). The commit is
# fetched from DataDog/iceberg-rust, which mirrors upstream; the workspace [patch] points
# datafusion-iceberg at it too so FileScanTask is one type.
iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg.git", rev = "5398be196e47ef42543f6e9c72d15cad898e5143" }
futures = "0.3"
iceberg = { git = "https://github.com/apache/iceberg-rust.git", rev = "0d5eb123495f765ea151e78c8964919d554d4b98" }
iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust.git", rev = "0d5eb123495f765ea151e78c8964919d554d4b98", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] }
iceberg = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102" }
iceberg-storage-opendal = { git = "https://github.com/DataDog/iceberg-rust.git", rev = "13f84f93f1a09619f473b788b2a88ef1da0c7102", default-features = false, features = ["opendal-fs", "opendal-s3", "opendal-gcs"] }
datafusion-distributed = { path = "../distributed", features = ["sql"] }
delegate = "0.13"
prost = "0.14.1"
Expand Down
6 changes: 4 additions & 2 deletions datafusion/distributed-iceberg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,10 @@ Snapshot distinct-count estimates are retained as an inexact fallback, capped by
selected-file row counts when known. They are not recomputed for the selected
files and can overestimate the filtered result. Sum statistics remain unknown.

Another manifest pass is needed because the pinned Iceberg API does not retain
column metrics in file tasks. Scan task counts continue to use whole-file bytes.
The metrics come from the planned file tasks themselves: the scan asks Iceberg to
retain the statistics columns' metrics (`TableScanBuilder::with_column_stats`, from
DataDog's iceberg-rust fork), so no manifest is read a second time. They are dropped
from the tasks once summarized. Scan task counts continue to use whole-file bytes.

This integration branch also exports `datafusion_distributed::ColumnByteStatisticsProvider`.
To preserve available column-byte estimates through grouping keys and inner joins,
Expand Down
2 changes: 1 addition & 1 deletion datafusion/distributed-iceberg/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ extensions_options! {
pub row_group_filtering_enabled: bool, default = true
/// Whether to apply row-level selections while reading Parquet files.
pub row_selection_enabled: bool, default = false
/// Read column statistics during planning (may require another manifest pass).
/// Read column statistics during planning (retained from the files selected during planning).
/// Eager scans aggregate selected-file metrics and retain inexact snapshot NDV.
pub column_stats_enabled: bool, default = false
/// Discover and retain file tasks during physical planning instead of execution.
Expand Down
40 changes: 35 additions & 5 deletions datafusion/distributed-iceberg/src/eager/planned_files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ use datafusion::common::{Result, Statistics};
use datafusion::execution::TaskContext;
use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};
use futures::TryStreamExt;
use iceberg::scan::{FileScanTask, TableScan};
use iceberg::scan::{FileScanTask, FileScanTaskColumnStats, TableScan};
use iceberg::spec::{Datum, PrimitiveLiteral};

use crate::common::df_err;

Expand Down Expand Up @@ -86,12 +87,41 @@ fn retained_size(task: &FileScanTask) -> usize {
+ size_of_val(task.project_field_ids())
+ size_of_val(task.deletes())
+ task.key_metadata().map_or(0, <[u8]>::len);
if let Some(stats) = task.column_stats() {
bytes = bytes.saturating_add(column_stats_size(stats));
}
for delete in task.deletes() {
bytes = bytes
.saturating_add(delete.file_path.len())
.saturating_add(delete.referenced_data_file.as_deref().map_or(0, str::len))
.saturating_add(delete.equality_ids.as_deref().map_or(0, size_of_val))
.saturating_add(delete.key_metadata.as_deref().map_or(0, <[u8]>::len));
.saturating_add(delete.file_path().len())
.saturating_add(delete.referenced_data_file().map_or(0, str::len))
.saturating_add(delete.equality_ids().map_or(0, size_of_val))
.saturating_add(delete.key_metadata().map_or(0, <[u8]>::len));
}
bytes
}

/// Approximate heap size of the metrics a task retains for column statistics.
fn column_stats_size(stats: &FileScanTaskColumnStats) -> usize {
let counts = [
stats.column_sizes(),
stats.value_counts(),
stats.null_value_counts(),
stats.nan_value_counts(),
]
.iter()
.map(|counts| counts.len() * size_of::<(i32, u64)>())
.sum::<usize>();
let bounds = [stats.lower_bounds(), stats.upper_bounds()]
.iter()
.flat_map(|bounds| bounds.values())
.map(|bound| {
size_of::<(i32, Datum)>()
+ match bound.literal() {
PrimitiveLiteral::String(value) => value.len(),
PrimitiveLiteral::Binary(value) => value.len(),
_ => 0,
}
})
.sum::<usize>();
size_of::<FileScanTaskColumnStats>() + counts + bounds
}
166 changes: 94 additions & 72 deletions datafusion/distributed-iceberg/src/table_provider/column_statistics.rs
Original file line number Diff line number Diff line change
@@ -1,23 +1,48 @@
use crate::common::df_err;
use datafusion::common::runtime::JoinSet;
use datafusion::common::stats::Precision;
use datafusion::common::{ColumnStatistics, HashMap, HashSet, Result, exec_err};
use datafusion::common::{ColumnStatistics, HashMap, Result, exec_err};
use datafusion::error::DataFusionError;
use datafusion::scalar::ScalarValue;
use iceberg::puffin::APACHE_DATASKETCHES_THETA_V1;
use iceberg::spec::{
DataFile, Datum, ManifestContentType, ManifestList, NestedFieldRef, PrimitiveLiteral,
PrimitiveType, SnapshotRef, Type,
PrimitiveType, Schema, SnapshotRef, Type,
};
use iceberg::table::Table;
use std::sync::Arc;

use iceberg::scan::FileScanTask;
use iceberg::scan::{FileScanTask, FileScanTaskColumnStats};

/// Names of the primitive leaf columns under `fields`, in schema-dotted form. Passed to
/// `TableScanBuilder::with_column_stats` so each planned `FileScanTask` carries the file's
/// metrics for exactly the columns these statistics cover. Leaves that are not primitive
/// (variants) have no file metrics and are skipped.
pub(crate) fn column_stats_names(
schema: &Schema,
fields: &[NestedFieldRef],
) -> Result<Vec<String>> {
let mut names = vec![];
for field in fields {
let mut leaves = vec![];
leaves_of(field, &mut leaves);
for leaf in leaves {
if !matches!(leaf.field_type.as_ref(), Type::Primitive(_)) {
continue;
}
let Some(name) = schema.name_by_field_id(leaf.id) else {
return exec_err!("Iceberg column {} has no name in the scan schema", leaf.id);
};
names.push(name.to_string());
}
}
Ok(names)
}

/// Selected-file metrics, with snapshot NDV retained only as an inexact fallback.
/// FileScanTask does not retain manifest metrics, so this requires a metadata-only
/// second pass. Stop as soon as all selected files have been found.
pub(crate) async fn selected_column_statistics(
/// Metrics come from the `FileScanTask::column_stats` that the scan retained while
/// planning (see [`column_stats_names`]), so no manifest is read a second time.
pub(crate) fn selected_column_statistics(
table: &Table,
snapshot: &SnapshotRef,
fields: &[NestedFieldRef],
Expand All @@ -42,10 +67,6 @@ pub(crate) async fn selected_column_statistics(
// Full-file metrics cannot describe a byte-range task.
return Ok(vec![ColumnStatistics::new_unknown(); fields.len()]);
}
let mut remaining: HashSet<_> = files.iter().map(|task| task.data_file_path()).collect();
if remaining.len() != files.len() {
return exec_err!("Duplicate whole-file tasks in Iceberg column statistics lookup");
}
let field_ids: Vec<_> = fields
.iter()
.map(|field| {
Expand All @@ -55,51 +76,31 @@ pub(crate) async fn selected_column_statistics(
})
.collect();
let mut statistics = vec![None; fields.len()];
let manifests = table
.manifest_list_reader(snapshot)
.load()
.await
.map_err(df_err)?;
for manifest in manifests
.entries()
.iter()
.filter(|manifest| manifest.content == ManifestContentType::Data)
{
let manifest = table
.manifest_reader()
.read(manifest)
.await
.map_err(df_err)?;
for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) {
let file = entry.data_file();
if !remaining.remove(file.file_path()) {
continue;
}
for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) {
// Leaf metrics cannot describe a parent struct/list/map's value or null count.
let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) {
data_file_col_stats(file, field.id)
} else {
ColumnStatistics::new_unknown()
};
next.byte_size = ids
.iter()
.try_fold(0_usize, |total, id| {
let size = usize::try_from(*file.column_sizes().get(id)?).ok()?;
total.checked_add(size)
})
.map(Precision::Inexact)
.unwrap_or(Precision::Absent);
*acc = Some(merge_col_stats(acc.take(), next));
}
}
if remaining.is_empty() {
break;
for task in files {
let Some(file) = task.column_stats() else {
return exec_err!(
"Planned Iceberg file {} carries no column statistics",
task.data_file_path()
);
};
for ((acc, field), ids) in statistics.iter_mut().zip(fields).zip(&field_ids) {
// Leaf metrics cannot describe a parent struct/list/map's value or null count.
let mut next = if matches!(field.field_type.as_ref(), Type::Primitive(_)) {
task_col_stats(file, field.id)
} else {
ColumnStatistics::new_unknown()
};
next.byte_size = ids
.iter()
.try_fold(0_usize, |total, id| {
let size = usize::try_from(*file.column_sizes().get(id)?).ok()?;
total.checked_add(size)
})
.map(Precision::Inexact)
.unwrap_or(Precision::Absent);
*acc = Some(merge_col_stats(acc.take(), next));
}
}
if !remaining.is_empty() {
return exec_err!("Planned Iceberg files missing from snapshot manifests");
}
let rows = files.iter().try_fold(0_usize, |rows, task| {
rows.checked_add(usize::try_from(task.record_count()?).ok()?)
});
Expand All @@ -123,17 +124,23 @@ pub(crate) async fn selected_column_statistics(
}

fn leaf_ids(field: &NestedFieldRef, ids: &mut Vec<i32>) {
let mut leaves = vec![];
leaves_of(field, &mut leaves);
ids.extend(leaves.iter().map(|leaf| leaf.id));
}

fn leaves_of<'a>(field: &'a NestedFieldRef, leaves: &mut Vec<&'a NestedFieldRef>) {
match field.field_type.as_ref() {
Type::Primitive(_) | Type::Variant(_) => ids.push(field.id),
Type::Primitive(_) | Type::Variant(_) => leaves.push(field),
Type::Struct(fields) => {
for field in fields.fields() {
leaf_ids(field, ids);
leaves_of(field, leaves);
}
}
Type::List(list) => leaf_ids(&list.element_field, ids),
Type::List(list) => leaves_of(&list.element_field, leaves),
Type::Map(map) => {
leaf_ids(&map.key_field, ids);
leaf_ids(&map.value_field, ids);
leaves_of(&map.key_field, leaves);
leaves_of(&map.value_field, leaves);
}
}
}
Expand Down Expand Up @@ -279,28 +286,43 @@ fn merge_col_stats(acc: Option<ColumnStatistics>, next: ColumnStatistics) -> Col
}

fn data_file_col_stats(df: &DataFile, id: i32) -> ColumnStatistics {
file_col_stats(
df.null_value_counts().get(&id).copied(),
df.lower_bounds().get(&id),
df.upper_bounds().get(&id),
df.column_sizes().get(&id).copied(),
)
}

fn task_col_stats(stats: &FileScanTaskColumnStats, id: i32) -> ColumnStatistics {
file_col_stats(
stats.null_value_counts().get(&id).copied(),
stats.lower_bounds().get(&id),
stats.upper_bounds().get(&id),
stats.column_sizes().get(&id).copied(),
)
}

fn file_col_stats(
null_count: Option<u64>,
lower_bound: Option<&Datum>,
upper_bound: Option<&Datum>,
column_size: Option<u64>,
) -> ColumnStatistics {
ColumnStatistics {
null_count: df
.null_value_counts()
.get(&id)
.map(|n| Precision::Exact(*n as usize))
null_count: null_count
.map(|n| Precision::Exact(n as usize))
.unwrap_or(Precision::Absent),
min_value: df
.lower_bounds()
.get(&id)
min_value: lower_bound
.and_then(datum_to_scalar)
.map(Precision::Inexact)
.unwrap_or(Precision::Absent),
max_value: df
.upper_bounds()
.get(&id)
max_value: upper_bound
.and_then(datum_to_scalar)
.map(Precision::Inexact)
.unwrap_or(Precision::Absent),
byte_size: df
.column_sizes()
.get(&id)
.map(|n| Precision::Inexact(*n as usize))
byte_size: column_size
.map(|n| Precision::Inexact(n as usize))
.unwrap_or(Precision::Absent),
sum_value: Precision::Absent,
distinct_count: Precision::Absent,
Expand Down
Loading
Loading