From deb671be15d5d8e2fed31a1b3c0aaa59275ada1f Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 17 Nov 2026 08:51:27 -0500 Subject: [PATCH] refactor: tidy table provider constructors and stale names - Make IcebergStaticTableProvider::try_new_from_table and try_new_from_table_snapshot synchronous; neither awaited anything. - Replace IcebergMetadataTableProvider::new with a fallible try_new that converts the schema once, so TableProvider::schema no longer unwraps. - Replace the old IcebergTableProvider name in error messages and docs, and refresh other stale names in AGENTS.md. --- AGENTS.md | 8 ++--- crates/datafusion/src/metadata/table.rs | 28 ++++++++++------ crates/datafusion/src/read/predicate.rs | 5 +-- .../table_provider/catalog_table_provider.rs | 13 ++++---- .../datafusion/src/table_provider/factory.rs | 4 +-- .../table_provider/static_table_provider.rs | 32 ++++++------------- .../tests/integration_datafusion_test.rs | 7 ++-- 7 files changed, 44 insertions(+), 53 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dae9660..65481c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ Tests are self-contained (in-memory catalogs, temp dirs, checked-in metadata JSO - Names like `$snapshots`, `$manifests`, and `$history` resolve to `IcebergMetadataTableProvider`. - SQL `CREATE TABLE` and `DROP TABLE` reach the *synchronous* `SchemaProvider::register_table` and `deregister_table` methods, which call the async catalog through `spawn_blocking` + `block_on`. `register_table` rejects input tables that contain rows, so CTAS isn't supported. It auto-assigns field IDs and uses format V2 unless the schema needs V3. -### Table providers (`table/`) +### Table providers (`table_provider/`) - **`IcebergCatalogTableProvider`**: catalog-backed. It caches the Arrow schema at construction but reloads table metadata from the catalog on every `scan` and `insert_into`. INSERT supports append only. - **`IcebergStaticTableProvider`**: wraps a fixed `Table`, optionally pinned to a snapshot id for time travel. Read-only. @@ -71,11 +71,11 @@ Tests are self-contained (in-memory catalogs, temp dirs, checked-in metadata JSO ### Read path and filter pushdown -`IcebergTableScan` converts the projection to column names and the filters to a single Iceberg `Predicate` (`read/predicate.rs`). It applies `limit` in-stream, has one output partition, and delegates the actual read to iceberg's `TableScan::to_arrow()`. +`IcebergDataSource` converts the projection to column names and the filters to a single Iceberg `Predicate` (`read/predicate.rs`). It applies `limit` in-stream, has one output partition, and delegates the actual read to iceberg's `TableScan::to_arrow()`. Both providers report every filter as `Inexact`, so DataFusion re-applies the original filters after the scan. The pushed-down predicate therefore only has to be *implied by* the original filter. It may match extra rows but must never drop a row that matches. Unconvertible parts are dropped: an `AND` keeps whichever side converted, while an `OR` needs both sides. Preserve this soundness rule when extending `predicate.rs`; for example, it is why casts and some NaN arithmetic are deliberately not pushed down. Removing a cast is left to DataFusion's simplifier (`unwrap_cast_in_comparison`), which rewrites a comparison only when the result doesn't change. -### Write path (`IcebergTableProvider::insert_into`) +### Write path (`IcebergCatalogTableProvider::insert_into`) ``` input @@ -105,7 +105,7 @@ Library code returns `datafusion::error::Result`. Convert iceberg errors with `t - `test_encrypted_round_trip`: format V3, encrypted with in-memory KMS key `test-master-key` All other tables are created by `CREATE TABLE` inside the `.slt` files. -- Several `.slt` files assert `EXPLAIN` output, such as `IcebergTableScan projection:[...] predicate:[...] limit:[...]`. Changes to the scan's `DisplayAs`, to pushdown, or to the DataFusion version require updating those expectations by hand; the harness has no auto-complete mode. +- Several `.slt` files assert `EXPLAIN` output, such as `DataSourceExec: format=iceberg, projection=[...], predicate=..., fetch=...`. Changes to `IcebergDataSource::fmt_as`, to pushdown, or to the DataFusion version require updating those expectations by hand; the harness has no auto-complete mode. ## Dependencies diff --git a/crates/datafusion/src/metadata/table.rs b/crates/datafusion/src/metadata/table.rs index 3cf351d..73e0ee3 100644 --- a/crates/datafusion/src/metadata/table.rs +++ b/crates/datafusion/src/metadata/table.rs @@ -41,18 +41,13 @@ use crate::to_datafusion_error; pub struct IcebergMetadataTableProvider { table: Table, r#type: MetadataTableType, + schema: ArrowSchemaRef, } #[async_trait] impl TableProvider for IcebergMetadataTableProvider { fn schema(&self) -> ArrowSchemaRef { - let metadata_table = self.table.inspect(); - let schema = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().schema(), - MetadataTableType::Manifests => metadata_table.manifests().schema(), - MetadataTableType::History => metadata_table.history().schema(), - }; - schema_to_arrow_schema(&schema).unwrap().into() + self.schema.clone() } fn table_type(&self) -> TableType { @@ -75,11 +70,24 @@ impl TableProvider for IcebergMetadataTableProvider { impl IcebergMetadataTableProvider { /// Creates a provider for the `metadata_type` metadata table of an /// already loaded table. - pub fn new(table: Table, metadata_type: MetadataTableType) -> Self { - Self { + /// + /// # Errors + /// + /// Returns an error if the metadata table's schema has no Arrow equivalent. + pub fn try_new(table: Table, metadata_type: MetadataTableType) -> Result { + let metadata_table = table.inspect(); + let schema = match metadata_type { + MetadataTableType::Snapshots => metadata_table.snapshots().schema(), + MetadataTableType::Manifests => metadata_table.manifests().schema(), + MetadataTableType::History => metadata_table.history().schema(), + }; + let schema = + Arc::new(schema_to_arrow_schema(&schema).map_err(to_datafusion_error)?); + Ok(Self { table, r#type: metadata_type, - } + schema, + }) } /// The table whose metadata this provider reads. diff --git a/crates/datafusion/src/read/predicate.rs b/crates/datafusion/src/read/predicate.rs index 8e4197c..a6d43e8 100644 --- a/crates/datafusion/src/read/predicate.rs +++ b/crates/datafusion/src/read/predicate.rs @@ -257,8 +257,7 @@ fn scalar_function_to_iceberg_predicate( /// [`Reference`] such that `isnan(arg)` is logically equivalent to /// `isnan(reference)`. /// -/// Filter pushdown is reported as `Inexact` (see -/// [`IcebergTableProvider::supports_filters_pushdown`]), so DataFusion +/// Both table providers report filter pushdown as `Inexact`, so DataFusion /// re-applies the original predicate after scanning. We therefore only need the /// pushed-down predicate to be implied by the original filter (it may match /// extra rows, but must never drop a matching one). Every transformation handled @@ -272,8 +271,6 @@ fn scalar_function_to_iceberg_predicate( /// /// Multiplication/division by zero and `c / x` are intentionally rejected: e.g. /// `x * 0` is NaN when `x` is `±inf`, so it does not imply `x` is NaN. -/// -/// [`IcebergTableProvider::supports_filters_pushdown`]: crate::table_provider::IcebergCatalogTableProvider fn resolve_nan_preserving_reference(expr: &Expr) -> Option { match expr { Expr::Column(column) => Some(Reference::new(column.name())), diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs index b804f56..afed686 100644 --- a/crates/datafusion/src/table_provider/catalog_table_provider.rs +++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs @@ -100,7 +100,7 @@ impl IcebergCatalogTableProvider { .load_table(&self.table_ident) .await .map_err(to_datafusion_error)?; - Ok(IcebergMetadataTableProvider::new(table, r#type)) + IcebergMetadataTableProvider::try_new(table, r#type) } /// The catalog this provider loads its table from and commits through. @@ -159,11 +159,12 @@ impl TableProvider for IcebergCatalogTableProvider { &self, state: &dyn Session, input: Arc, - _insert_op: InsertOp, + insert_op: InsertOp, ) -> Result> { - if _insert_op != InsertOp::Append { + if insert_op != InsertOp::Append { return not_impl_err!( - "IcebergTableProvider supports only append inserts, got {_insert_op}" + "IcebergCatalogTableProvider supports only append inserts, \ + got {insert_op}" ); } @@ -383,11 +384,11 @@ mod tests { for (insert_op, expected_message) in [ ( InsertOp::Overwrite, - "IcebergTableProvider supports only append inserts, got Insert Overwrite", + "IcebergCatalogTableProvider supports only append inserts, got Insert Overwrite", ), ( InsertOp::Replace, - "IcebergTableProvider supports only append inserts, got Replace Into", + "IcebergCatalogTableProvider supports only append inserts, got Replace Into", ), ] { let input = diff --git a/crates/datafusion/src/table_provider/factory.rs b/crates/datafusion/src/table_provider/factory.rs index 161370d..5ddfe2d 100644 --- a/crates/datafusion/src/table_provider/factory.rs +++ b/crates/datafusion/src/table_provider/factory.rs @@ -31,7 +31,7 @@ use iceberg::table::StaticTable; use super::IcebergStaticTableProvider; use crate::to_datafusion_error; -/// A factory that implements DataFusion's `TableProviderFactory` to create `IcebergTableProvider` instances. +/// A factory that implements DataFusion's `TableProviderFactory` to create [`IcebergStaticTableProvider`] instances. /// /// # Example /// @@ -144,7 +144,7 @@ impl TableProviderFactory for IcebergTableProviderFactory { .await? .into_table(); - let provider = IcebergStaticTableProvider::try_new_from_table(table).await?; + let provider = IcebergStaticTableProvider::try_new_from_table(table)?; Ok(Arc::new(provider)) } diff --git a/crates/datafusion/src/table_provider/static_table_provider.rs b/crates/datafusion/src/table_provider/static_table_provider.rs index 5ebabc8..49f9897 100644 --- a/crates/datafusion/src/table_provider/static_table_provider.rs +++ b/crates/datafusion/src/table_provider/static_table_provider.rs @@ -40,7 +40,7 @@ use crate::read::IcebergDataSource; /// or when you want to avoid catalog overhead. /// /// For catalog-backed tables with write support and automatic refresh, use -/// [`IcebergTableProvider`](super::IcebergCatalogTableProvider) instead. +/// [`IcebergCatalogTableProvider`](super::IcebergCatalogTableProvider) instead. #[derive(Debug, Clone)] pub struct IcebergStaticTableProvider { /// The static table instance (never refreshed) @@ -55,7 +55,7 @@ impl IcebergStaticTableProvider { /// Creates a static provider from a table instance. /// /// Uses the table's current snapshot for all queries. Does not support write operations. - pub async fn try_new_from_table(table: Table) -> Result { + pub fn try_new_from_table(table: Table) -> Result { let schema = Arc::new( schema_to_arrow_schema(table.metadata().current_schema()) .map_err(to_datafusion_error)?, @@ -71,10 +71,7 @@ impl IcebergStaticTableProvider { /// /// Queries the specified snapshot for all operations. Useful for time-travel queries. /// Does not support write operations. - pub async fn try_new_from_table_snapshot( - table: Table, - snapshot_id: i64, - ) -> Result { + pub fn try_new_from_table_snapshot(table: Table, snapshot_id: i64) -> Result { let snapshot = table .metadata() .snapshot_by_id(snapshot_id) @@ -151,7 +148,7 @@ impl TableProvider for IcebergStaticTableProvider { ) -> Result> { not_impl_err!( "Write operations are not supported on IcebergStaticTableProvider. \ - Use IcebergTableProvider with a catalog for write support." + Use IcebergCatalogTableProvider with a catalog for write support." ) } } @@ -172,9 +169,7 @@ mod tests { async fn test_static_provider_from_table() { let table = get_test_table_from_metadata_file().await; let table_provider = - IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); + IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap(); let ctx = SessionContext::new(); ctx.register_table("mytable", Arc::new(table_provider)) .unwrap(); @@ -201,7 +196,6 @@ mod tests { table.clone(), snapshot_id, ) - .await .unwrap(); let ctx = SessionContext::new(); ctx.register_table("mytable", Arc::new(table_provider)) @@ -225,9 +219,7 @@ mod tests { async fn test_static_provider_rejects_writes() { let table = get_test_table_from_metadata_file().await; let table_provider = - IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); + IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap(); let ctx = SessionContext::new(); ctx.register_table("mytable", Arc::new(table_provider)) .unwrap(); @@ -249,9 +241,7 @@ mod tests { async fn test_static_provider_scan() { let table = get_test_table_from_metadata_file().await; let table_provider = - IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); + IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap(); let ctx = SessionContext::new(); ctx.register_table("mytable", Arc::new(table_provider)) .unwrap(); @@ -268,9 +258,7 @@ mod tests { let table = get_test_table_from_metadata_file().await; let table_provider = - IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); + IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap(); let ctx = SessionContext::new(); let state = ctx.state(); @@ -303,9 +291,7 @@ mod tests { let table = get_test_table_from_metadata_file().await; let table_provider = - IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); + IcebergStaticTableProvider::try_new_from_table(table.clone()).unwrap(); let ctx = SessionContext::new(); let state = ctx.state(); diff --git a/crates/datafusion/tests/integration_datafusion_test.rs b/crates/datafusion/tests/integration_datafusion_test.rs index 42d350a..26ddf01 100644 --- a/crates/datafusion/tests/integration_datafusion_test.rs +++ b/crates/datafusion/tests/integration_datafusion_test.rs @@ -1008,8 +1008,7 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { let table = client.load_table(&ident).await?; let snapshot_id = table.metadata().current_snapshot_id().unwrap(); let pinned = - IcebergStaticTableProvider::try_new_from_table_snapshot(table, snapshot_id) - .await?; + IcebergStaticTableProvider::try_new_from_table_snapshot(table, snapshot_id)?; assert_eq!(pinned.snapshot_id(), Some(snapshot_id)); ctx.register_table("pinned", Arc::new(pinned.clone()))?; // A later write, so a scan reading the current snapshot rather than the @@ -1103,10 +1102,10 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { let provider = metadata_scan.provider(); assert_eq!(provider.table().identifier(), &ident); let rebuilt = IcebergMetadataDataSource::try_new( - IcebergMetadataTableProvider::new( + IcebergMetadataTableProvider::try_new( provider.table().clone(), provider.metadata_type().clone(), - ), + )?, None, )?; let batches = run_batches(metadata_exec, &ctx).await?;