diff --git a/AGENTS.md b/AGENTS.md
index dae9660..65481c6 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -63,7 +63,7 @@ Tests are self-contained (in-memory catalogs, temp dirs, checked-in metadata JSO
- Names like `
$snapshots`, `$manifests`, and `$history` resolve to `IcebergMetadataTableProvider`.
- SQL `CREATE TABLE` and `DROP TABLE` reach the *synchronous* `SchemaProvider::register_table` and `deregister_table` methods, which call the async catalog through `spawn_blocking` + `block_on`. `register_table` rejects input tables that contain rows, so CTAS isn't supported. It auto-assigns field IDs and uses format V2 unless the schema needs V3.
-### Table providers (`table/`)
+### Table providers (`table_provider/`)
- **`IcebergCatalogTableProvider`**: catalog-backed. It caches the Arrow schema at construction but reloads table metadata from the catalog on every `scan` and `insert_into`. INSERT supports append only.
- **`IcebergStaticTableProvider`**: wraps a fixed `Table`, optionally pinned to a snapshot id for time travel. Read-only.
@@ -71,11 +71,11 @@ Tests are self-contained (in-memory catalogs, temp dirs, checked-in metadata JSO
### Read path and filter pushdown
-`IcebergTableScan` converts the projection to column names and the filters to a single Iceberg `Predicate` (`read/predicate.rs`). It applies `limit` in-stream, has one output partition, and delegates the actual read to iceberg's `TableScan::to_arrow()`.
+`IcebergDataSource` converts the projection to column names and the filters to a single Iceberg `Predicate` (`read/predicate.rs`). It applies `limit` in-stream, has one output partition, and delegates the actual read to iceberg's `TableScan::to_arrow()`.
Both providers report every filter as `Inexact`, so DataFusion re-applies the original filters after the scan. The pushed-down predicate therefore only has to be *implied by* the original filter. It may match extra rows but must never drop a row that matches. Unconvertible parts are dropped: an `AND` keeps whichever side converted, while an `OR` needs both sides. Preserve this soundness rule when extending `predicate.rs`; for example, it is why casts and some NaN arithmetic are deliberately not pushed down. Removing a cast is left to DataFusion's simplifier (`unwrap_cast_in_comparison`), which rewrites a comparison only when the result doesn't change.
-### Write path (`IcebergTableProvider::insert_into`)
+### Write path (`IcebergCatalogTableProvider::insert_into`)
```
input
@@ -105,7 +105,7 @@ Library code returns `datafusion::error::Result`. Convert iceberg errors with `t
- `test_encrypted_round_trip`: format V3, encrypted with in-memory KMS key `test-master-key`
All other tables are created by `CREATE TABLE` inside the `.slt` files.
-- Several `.slt` files assert `EXPLAIN` output, such as `IcebergTableScan projection:[...] predicate:[...] limit:[...]`. Changes to the scan's `DisplayAs`, to pushdown, or to the DataFusion version require updating those expectations by hand; the harness has no auto-complete mode.
+- Several `.slt` files assert `EXPLAIN` output, such as `DataSourceExec: format=iceberg, projection=[...], predicate=..., fetch=...`. Changes to `IcebergDataSource::fmt_as`, to pushdown, or to the DataFusion version require updating those expectations by hand; the harness has no auto-complete mode.
## Dependencies
diff --git a/crates/datafusion/src/metadata/table.rs b/crates/datafusion/src/metadata/table.rs
index 3cf351d..73e0ee3 100644
--- a/crates/datafusion/src/metadata/table.rs
+++ b/crates/datafusion/src/metadata/table.rs
@@ -41,18 +41,13 @@ use crate::to_datafusion_error;
pub struct IcebergMetadataTableProvider {
table: Table,
r#type: MetadataTableType,
+ schema: ArrowSchemaRef,
}
#[async_trait]
impl TableProvider for IcebergMetadataTableProvider {
fn schema(&self) -> ArrowSchemaRef {
- let metadata_table = self.table.inspect();
- let schema = match self.r#type {
- MetadataTableType::Snapshots => metadata_table.snapshots().schema(),
- MetadataTableType::Manifests => metadata_table.manifests().schema(),
- MetadataTableType::History => metadata_table.history().schema(),
- };
- schema_to_arrow_schema(&schema).unwrap().into()
+ self.schema.clone()
}
fn table_type(&self) -> TableType {
@@ -75,11 +70,24 @@ impl TableProvider for IcebergMetadataTableProvider {
impl IcebergMetadataTableProvider {
/// Creates a provider for the `metadata_type` metadata table of an
/// already loaded table.
- pub fn new(table: Table, metadata_type: MetadataTableType) -> Self {
- Self {
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the metadata table's schema has no Arrow equivalent.
+ pub fn try_new(table: Table, metadata_type: MetadataTableType) -> Result {
+ let metadata_table = table.inspect();
+ let schema = match metadata_type {
+ MetadataTableType::Snapshots => metadata_table.snapshots().schema(),
+ MetadataTableType::Manifests => metadata_table.manifests().schema(),
+ MetadataTableType::History => metadata_table.history().schema(),
+ };
+ let schema =
+ Arc::new(schema_to_arrow_schema(&schema).map_err(to_datafusion_error)?);
+ Ok(Self {
table,
r#type: metadata_type,
- }
+ schema,
+ })
}
/// The table whose metadata this provider reads.
diff --git a/crates/datafusion/src/read/predicate.rs b/crates/datafusion/src/read/predicate.rs
index 8e4197c..a6d43e8 100644
--- a/crates/datafusion/src/read/predicate.rs
+++ b/crates/datafusion/src/read/predicate.rs
@@ -257,8 +257,7 @@ fn scalar_function_to_iceberg_predicate(
/// [`Reference`] such that `isnan(arg)` is logically equivalent to
/// `isnan(reference)`.
///
-/// Filter pushdown is reported as `Inexact` (see
-/// [`IcebergTableProvider::supports_filters_pushdown`]), so DataFusion
+/// Both table providers report filter pushdown as `Inexact`, so DataFusion
/// re-applies the original predicate after scanning. We therefore only need the
/// pushed-down predicate to be implied by the original filter (it may match
/// extra rows, but must never drop a matching one). Every transformation handled
@@ -272,8 +271,6 @@ fn scalar_function_to_iceberg_predicate(
///
/// Multiplication/division by zero and `c / x` are intentionally rejected: e.g.
/// `x * 0` is NaN when `x` is `±inf`, so it does not imply `x` is NaN.
-///
-/// [`IcebergTableProvider::supports_filters_pushdown`]: crate::table_provider::IcebergCatalogTableProvider
fn resolve_nan_preserving_reference(expr: &Expr) -> Option {
match expr {
Expr::Column(column) => Some(Reference::new(column.name())),
diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs
index b804f56..afed686 100644
--- a/crates/datafusion/src/table_provider/catalog_table_provider.rs
+++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs
@@ -100,7 +100,7 @@ impl IcebergCatalogTableProvider {
.load_table(&self.table_ident)
.await
.map_err(to_datafusion_error)?;
- Ok(IcebergMetadataTableProvider::new(table, r#type))
+ IcebergMetadataTableProvider::try_new(table, r#type)
}
/// The catalog this provider loads its table from and commits through.
@@ -159,11 +159,12 @@ impl TableProvider for IcebergCatalogTableProvider {
&self,
state: &dyn Session,
input: Arc,
- _insert_op: InsertOp,
+ insert_op: InsertOp,
) -> Result> {
- if _insert_op != InsertOp::Append {
+ if insert_op != InsertOp::Append {
return not_impl_err!(
- "IcebergTableProvider supports only append inserts, got {_insert_op}"
+ "IcebergCatalogTableProvider supports only append inserts, \
+ got {insert_op}"
);
}
@@ -383,11 +384,11 @@ mod tests {
for (insert_op, expected_message) in [
(
InsertOp::Overwrite,
- "IcebergTableProvider supports only append inserts, got Insert Overwrite",
+ "IcebergCatalogTableProvider supports only append inserts, got Insert Overwrite",
),
(
InsertOp::Replace,
- "IcebergTableProvider supports only append inserts, got Replace Into",
+ "IcebergCatalogTableProvider supports only append inserts, got Replace Into",
),
] {
let input =
diff --git a/crates/datafusion/src/table_provider/factory.rs b/crates/datafusion/src/table_provider/factory.rs
index 161370d..5ddfe2d 100644
--- a/crates/datafusion/src/table_provider/factory.rs
+++ b/crates/datafusion/src/table_provider/factory.rs
@@ -31,7 +31,7 @@ use iceberg::table::StaticTable;
use super::IcebergStaticTableProvider;
use crate::to_datafusion_error;
-/// A factory that implements DataFusion's `TableProviderFactory` to create `IcebergTableProvider` instances.
+/// A factory that implements DataFusion's `TableProviderFactory` to create [`IcebergStaticTableProvider`] instances.
///
/// # Example
///
@@ -144,7 +144,7 @@ impl TableProviderFactory for IcebergTableProviderFactory {
.await?
.into_table();
- let provider = IcebergStaticTableProvider::try_new_from_table(table).await?;
+ let provider = IcebergStaticTableProvider::try_new_from_table(table)?;
Ok(Arc::new(provider))
}
diff --git a/crates/datafusion/src/table_provider/static_table_provider.rs b/crates/datafusion/src/table_provider/static_table_provider.rs
index 5ebabc8..49f9897 100644
--- a/crates/datafusion/src/table_provider/static_table_provider.rs
+++ b/crates/datafusion/src/table_provider/static_table_provider.rs
@@ -40,7 +40,7 @@ use crate::read::IcebergDataSource;
/// or when you want to avoid catalog overhead.
///
/// For catalog-backed tables with write support and automatic refresh, use
-/// [`IcebergTableProvider`](super::IcebergCatalogTableProvider) instead.
+/// [`IcebergCatalogTableProvider`](super::IcebergCatalogTableProvider) instead.
#[derive(Debug, Clone)]
pub struct IcebergStaticTableProvider {
/// The static table instance (never refreshed)
@@ -55,7 +55,7 @@ impl IcebergStaticTableProvider {
/// Creates a static provider from a table instance.
///
/// Uses the table's current snapshot for all queries. Does not support write operations.
- pub async fn try_new_from_table(table: Table) -> Result {
+ pub fn try_new_from_table(table: Table) -> Result {
let schema = Arc::new(
schema_to_arrow_schema(table.metadata().current_schema())
.map_err(to_datafusion_error)?,
@@ -71,10 +71,7 @@ impl IcebergStaticTableProvider {
///
/// Queries the specified snapshot for all operations. Useful for time-travel queries.
/// Does not support write operations.
- pub async fn try_new_from_table_snapshot(
- table: Table,
- snapshot_id: i64,
- ) -> Result {
+ pub fn try_new_from_table_snapshot(table: Table, snapshot_id: i64) -> Result {
let snapshot = table
.metadata()
.snapshot_by_id(snapshot_id)
@@ -151,7 +148,7 @@ impl TableProvider for IcebergStaticTableProvider {
) -> Result> {
not_impl_err!(
"Write operations are not supported on IcebergStaticTableProvider. \
- Use IcebergTableProvider with a catalog for write support."
+ Use IcebergCatalogTableProvider with a catalog for write support."
)
}
}
@@ -172,9 +169,7 @@ mod tests {
async fn test_static_provider_from_table() {
let table = get_test_table_from_metadata_file().await;
let table_provider =
- IcebergStaticTableProvider::try_new_from_table(table.clone())
- .await
- .unwrap();
+ IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap();
let ctx = SessionContext::new();
ctx.register_table("mytable", Arc::new(table_provider))
.unwrap();
@@ -201,7 +196,6 @@ mod tests {
table.clone(),
snapshot_id,
)
- .await
.unwrap();
let ctx = SessionContext::new();
ctx.register_table("mytable", Arc::new(table_provider))
@@ -225,9 +219,7 @@ mod tests {
async fn test_static_provider_rejects_writes() {
let table = get_test_table_from_metadata_file().await;
let table_provider =
- IcebergStaticTableProvider::try_new_from_table(table.clone())
- .await
- .unwrap();
+ IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap();
let ctx = SessionContext::new();
ctx.register_table("mytable", Arc::new(table_provider))
.unwrap();
@@ -249,9 +241,7 @@ mod tests {
async fn test_static_provider_scan() {
let table = get_test_table_from_metadata_file().await;
let table_provider =
- IcebergStaticTableProvider::try_new_from_table(table.clone())
- .await
- .unwrap();
+ IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap();
let ctx = SessionContext::new();
ctx.register_table("mytable", Arc::new(table_provider))
.unwrap();
@@ -268,9 +258,7 @@ mod tests {
let table = get_test_table_from_metadata_file().await;
let table_provider =
- IcebergStaticTableProvider::try_new_from_table(table.clone())
- .await
- .unwrap();
+ IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap();
let ctx = SessionContext::new();
let state = ctx.state();
@@ -303,9 +291,7 @@ mod tests {
let table = get_test_table_from_metadata_file().await;
let table_provider =
- IcebergStaticTableProvider::try_new_from_table(table.clone())
- .await
- .unwrap();
+ IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap();
let ctx = SessionContext::new();
let state = ctx.state();
diff --git a/crates/datafusion/tests/integration_datafusion_test.rs b/crates/datafusion/tests/integration_datafusion_test.rs
index 42d350a..26ddf01 100644
--- a/crates/datafusion/tests/integration_datafusion_test.rs
+++ b/crates/datafusion/tests/integration_datafusion_test.rs
@@ -1008,8 +1008,7 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> {
let table = client.load_table(&ident).await?;
let snapshot_id = table.metadata().current_snapshot_id().unwrap();
let pinned =
- IcebergStaticTableProvider::try_new_from_table_snapshot(table, snapshot_id)
- .await?;
+ IcebergStaticTableProvider::try_new_from_table_snapshot(table, snapshot_id)?;
assert_eq!(pinned.snapshot_id(), Some(snapshot_id));
ctx.register_table("pinned", Arc::new(pinned.clone()))?;
// A later write, so a scan reading the current snapshot rather than the
@@ -1103,10 +1102,10 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> {
let provider = metadata_scan.provider();
assert_eq!(provider.table().identifier(), &ident);
let rebuilt = IcebergMetadataDataSource::try_new(
- IcebergMetadataTableProvider::new(
+ IcebergMetadataTableProvider::try_new(
provider.table().clone(),
provider.metadata_type().clone(),
- ),
+ )?,
None,
)?;
let batches = run_batches(metadata_exec, &ctx).await?;