Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions crates/datafusion/src/config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

use std::sync::LazyLock;

use datafusion::common::{config_namespace, extensions_options};
use datafusion::config::ConfigExtension;
use datafusion::prelude::SessionConfig;

extensions_options! {
/// Session options for Iceberg tables, under `iceberg.<group>.<option>`.
///
/// Register this extension before setting Iceberg options:
/// ```
/// use datafusion::prelude::SessionConfig;
/// use iceberg_datafusion::IcebergDataFusionConfig;
///
/// let config = SessionConfig::new()
/// .with_option_extension(IcebergDataFusionConfig::default())
/// .set_bool("iceberg.execution.eager_file_planning", true);
/// ```
pub struct IcebergDataFusionConfig {
/// Options for executing table scans.
pub execution: IcebergExecutionConfig, default = IcebergExecutionConfig::default()
}
}

config_namespace! {
/// Options for executing Iceberg scans, under `iceberg.execution`.
pub struct IcebergExecutionConfig {
/// Discover file tasks in TableProvider::scan instead of at execution, and
/// split them across up to the session's target partitions.
pub eager_file_planning: bool, default = false
}
}

impl ConfigExtension for IcebergDataFusionConfig {
const PREFIX: &'static str = "iceberg";
}

static DEFAULT: LazyLock<IcebergDataFusionConfig> =
LazyLock::new(IcebergDataFusionConfig::default);

impl IcebergDataFusionConfig {
pub fn from_config_or_default(cfg: &SessionConfig) -> &Self {
match cfg.options().extensions.get::<Self>() {
Some(v) => v,
None => &DEFAULT,
}
}
}
3 changes: 3 additions & 0 deletions crates/datafusion/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
mod catalog;
pub use catalog::*;

mod config;
pub use config::{IcebergDataFusionConfig, IcebergExecutionConfig};

mod error;
pub use error::*;

Expand Down
78 changes: 78 additions & 0 deletions crates/datafusion/src/read/file_scan_task_provider/eager.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

use datafusion::common::{Result, exec_err};
use datafusion::physical_plan::Partitioning;
use futures::StreamExt;
use iceberg::scan::FileScanTask;
use std::fmt::Formatter;
use std::sync::Arc;

use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext};

/// Files discovered before execution, in groups that each become one output
/// partition. Group order and empty groups are retained.
#[derive(Debug)]
pub struct EagerFileScanTaskProvider {
pub(crate) file_scan_tasks: Vec<Vec<FileScanTask>>,
}

impl EagerFileScanTaskProvider {
/// Reads `file_scan_tasks`, one output partition per group.
///
/// # Errors
///
/// Returns an error if there are no groups. Empty groups are allowed.
pub fn new(file_scan_tasks: Vec<Vec<FileScanTask>>) -> Self {
Self { file_scan_tasks }
}

/// All file tasks, in partition order and then group order.
pub fn tasks(&self) -> impl Iterator<Item = &FileScanTask> {
self.file_scan_tasks.iter().flatten()
}

/// The file tasks of each partition.
pub fn groups(&self) -> &[Vec<FileScanTask>] {
&self.file_scan_tasks
}
}

impl FileScanTaskProvider for EagerFileScanTaskProvider {
fn partitioning(&self) -> Partitioning {
Partitioning::UnknownPartitioning(self.groups().len())
}

fn open(
self: Arc<Self>,
partition: usize,
_context: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream> {
let Some(count) = self.groups().get(partition).map(Vec::len) else {
return exec_err!("Invalid Iceberg scan partition {partition}");
};
// Clone each task as it's read; holding `self` keeps the groups alive
// after the data source is dropped.
Ok(futures::stream::iter(0..count)
.map(move |offset| Ok(self.groups()[partition][offset].clone()))
.boxed())
}

fn fmt_as(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, ", planned_files={}", self.tasks().count())
}
}
80 changes: 80 additions & 0 deletions crates/datafusion/src/read/file_scan_task_provider/lazy.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::fmt::Formatter;
use std::sync::Arc;

use datafusion::common::Result;
use datafusion::physical_plan::Partitioning;
use futures::TryStreamExt;
use iceberg::expr::Predicate;

use super::provider::{FileScanTaskProvider, FileScanTaskProviderContext};
use crate::to_datafusion_error;

/// Discovers files while executing, in a single partition: the files of a
/// snapshot that may match a predicate.
#[derive(Debug, Clone)]
pub struct LazyFileScanTaskProvider {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we might be able to push this even further

  struct LazyFileScanTaskProvider {
      table: Table,
      snapshot_id: Option<i64>,
      projection: Vec<String>,
      predicate: Option<Predicate>,
  }

and

  struct EagerFileScanTaskProvider {
      groups: Vec<Vec<FileScanTask>>,
  }

Such that we can remove some of the logic from the data source, and it's builder.

   let discovery = Arc::new(LazyFileScanTaskProvider {
       table,
       snapshot_id,
       projection,
       predicate,
   });

   let provider: Arc<dyn FileScanTaskProvider> =
       if iceberg_config.execution.eager_file_planning {
           Arc::new(
               EagerFileScanTaskProvider::plan(discovery, context).await?,
           )
       } else {
           discovery
       };

   let source = IcebergDataSourceBuilder::new(schema, file_io, reader)
       .with_file_scan_task_provider(provider)
       .with_fetch(limit)
       .build()?;

Something like this, isolates the async out of the data source builder and removes the Table from the data source.

pub(crate) snapshot_id: Option<i64>,
pub(crate) projection: Vec<String>,
pub(crate) predicate: Option<Predicate>,
}

impl LazyFileScanTaskProvider {
/// Snapshot to scan, or the table's current one if `None`.
pub fn snapshot_id(&self) -> Option<i64> {
self.snapshot_id
}

/// Filter used to skip files, if any.
pub fn predicate(&self) -> Option<&Predicate> {
self.predicate.as_ref()
}
}

impl FileScanTaskProvider for LazyFileScanTaskProvider {
fn partitioning(&self) -> Partitioning {
Partitioning::UnknownPartitioning(1)
}

fn open(
self: Arc<Self>,
_partition: usize,
ctx: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream> {
let scan = match self.snapshot_id {
Some(snapshot_id) => ctx.table.scan().snapshot_id(snapshot_id),
None => ctx.table.scan(),
};
let mut scan = scan.select(&self.projection);
if let Some(predicate) = &self.predicate {
scan = scan.with_filter(predicate.clone());
}
let scan = scan.build().map_err(to_datafusion_error)?;
// Files are discovered when the stream is first polled, not here.
Ok(Box::pin(
futures::stream::once(async move { scan.plan_files().await }).try_flatten(),
))
}

fn fmt_as(&self, f: &mut Formatter) -> std::fmt::Result {
if let Some(predicate) = &self.predicate {
write!(f, ", predicate={predicate}")?;
}
Ok(())
}
}
23 changes: 23 additions & 0 deletions crates/datafusion/src/read/file_scan_task_provider/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
mod eager;
mod lazy;
mod provider;

pub use eager::EagerFileScanTaskProvider;
pub use lazy::LazyFileScanTaskProvider;
pub use provider::{FileScanTaskProvider, FileScanTaskProviderContext};
66 changes: 66 additions & 0 deletions crates/datafusion/src/read/file_scan_task_provider/provider.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::any::Any;
use std::fmt::{Debug, Formatter};
use std::sync::Arc;

use datafusion::common::Result;
use datafusion::execution::TaskContext;
use datafusion::physical_plan::Partitioning;
use iceberg::table::Table;

/// Supplies the file tasks an [`IcebergDataSource`](crate::read::IcebergDataSource)
/// reads, split into output partitions.
///
/// The data source owns reading, limits and metrics; implementations only
/// decide which files each partition reads and when they are discovered.
/// Implementations are `Any`, so consumers such as codecs can downcast a
/// source's [`file_scan_task_provider`](crate::read::IcebergDataSource::file_scan_task_provider)
/// to a concrete type:
///
/// ```
/// # use std::any::Any;
/// # use iceberg_datafusion::read::{IcebergDataSource, EagerFileScanTaskProvider};
/// fn planned_files(source: &IcebergDataSource) -> Option<&EagerFileScanTaskProvider> {
/// (source.file_scan_task_provider() as &dyn Any).downcast_ref::<EagerFileScanTaskProvider>()
/// }
/// ```
pub trait FileScanTaskProvider: Any + Debug + Send + Sync {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@alexanderbianchi this could be hour hooking point in DFD for injecting our WorkUnitFeed, so the idea here is to kill two birds with one stone:

  • providing polymorphism that supports both eager and lazy execution modes
  • provide a public hooking point for injecting custom FileScanTask streams, like DFD's WorkUnitFeed

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes that was the idea in #107. Similar API, I like yours returning the partitioning instead of the count more. I don't know I'd considered it also a solution for the eager path (or if I did, I had forgotten 😆)

pub trait FileScanTaskSource: Clone + Debug + Send + Sync + 'static {
    /// Number of local output partitions, not a worker or distributed task count.
    fn partition_count(&self) -> usize;

    /// Opens a partition's stream during execution, without collecting it.
    ///
    /// Discovery can start here or when the stream is polled. Calls on one
    /// instance may coordinate discovery; a repeated claim of the same partition
    /// may be rejected. Implementations must not block waiting for tasks here.
    fn tasks(
        &self,
        partition: usize,
        context: Arc<TaskContext>,
    ) -> Result<FileScanTaskStream>;

    /// Appends task-source details to the data source's explain output.
    fn fmt_as(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
        Ok(())
    }
}

/// How the data source's output is partitioned, with at least one
/// partition. Any expressions refer to columns of the data source's
/// projected schema.
fn partitioning(&self) -> Partitioning;

/// Streams the file tasks of `partition`, which is below the
/// [`Self::partitioning`] partition count.
fn open(
self: Arc<Self>,
partition: usize,
context: FileScanTaskProviderContext,
) -> Result<iceberg::scan::FileScanTaskStream>;

/// Appends details to the data source's `EXPLAIN` line, each starting
/// with `", "`. Writes nothing by default.
fn fmt_as(&self, _f: &mut Formatter) -> std::fmt::Result {
Ok(())
}
}

pub struct FileScanTaskProviderContext<'a> {
pub task_context: &'a Arc<TaskContext>,
pub table: &'a Table,

@alexanderbianchi alexanderbianchi Nov 17, 2026 •

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like if we also had a way to construct from not a table.

IMO there should be a way to pass in just FileScanTask and construct a data source,

  • FileScanTask is a public API in iceberg rust
  • Comet also passes FileScanTasks directly to DF, bypassing the Iceberg Catalog->Table->manifset planning path.
  • The Table in iceberg rust contains a lot. (FileIO, Cache, runtime)
  • It's needed for reconstruction, although I amm supportive of offering one codec from this library that could be shared between DFD / Ballista.

}
5 changes: 5 additions & 0 deletions crates/datafusion/src/read/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,13 @@
// specific language governing permissions and limitations
// under the License.

mod file_scan_task_provider;
pub(crate) mod predicate;
pub(crate) mod scan;

pub use file_scan_task_provider::{
EagerFileScanTaskProvider, FileScanTaskProvider, FileScanTaskProviderContext,
LazyFileScanTaskProvider,
};
pub use predicate::convert_filters_to_predicate;
pub use scan::{IcebergDataSource, IcebergDataSourceBuilder};
Loading
Loading