From e6583d269689fbfab1c2d2ce49f3b7272bb44ff2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 17 Nov 2026 15:20:49 +0100 Subject: [PATCH] feat: abstract Iceberg file discovery behind a FileScanTaskProvider trait --- crates/datafusion/src/config.rs | 65 ++++ crates/datafusion/src/lib.rs | 3 + .../src/read/file_scan_task_provider/eager.rs | 78 ++++ .../src/read/file_scan_task_provider/lazy.rs | 80 ++++ .../src/read/file_scan_task_provider/mod.rs | 23 ++ .../read/file_scan_task_provider/provider.rs | 66 ++++ crates/datafusion/src/read/mod.rs | 5 + crates/datafusion/src/read/scan.rs | 346 ++++++++++-------- .../table_provider/catalog_table_provider.rs | 27 +- .../table_provider/static_table_provider.rs | 20 +- .../tests/integration_datafusion_test.rs | 33 +- 11 files changed, 580 insertions(+), 166 deletions(-) create mode 100644 crates/datafusion/src/config.rs create mode 100644 crates/datafusion/src/read/file_scan_task_provider/eager.rs create mode 100644 crates/datafusion/src/read/file_scan_task_provider/lazy.rs create mode 100644 crates/datafusion/src/read/file_scan_task_provider/mod.rs create mode 100644 crates/datafusion/src/read/file_scan_task_provider/provider.rs diff --git a/crates/datafusion/src/config.rs b/crates/datafusion/src/config.rs new file mode 100644 index 0000000..ec45866 --- /dev/null +++ b/crates/datafusion/src/config.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::LazyLock; + +use datafusion::common::{config_namespace, extensions_options}; +use datafusion::config::ConfigExtension; +use datafusion::prelude::SessionConfig; + +extensions_options! { + /// Session options for Iceberg tables, under `iceberg..