Skip to content

feat(writer): apply per-column parquet bloom filter write properties #3148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alessandro-nori wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(writer): apply per-column parquet bloom filter write properties #3148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9bc3cf9
feat: apply parquet bloom filter write properties
alessandro-nori Sep 4, 2026
b1fb9ec
fix: validate bloom props, reuse path index
alessandro-nori Sep 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions crates/iceberg/public-api.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2839,6 +2839,10 @@ pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSION
pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize
pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str
pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX: &'static str
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT: f64
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX: &'static str
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX: &'static str
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED: &'static str
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool
pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str
Expand Down Expand Up @@ -2890,6 +2894,9 @@ pub fn iceberg::spec::TableProperties<'properties>::max_snapshot_age_ms(&self) -
pub fn iceberg::spec::TableProperties<'properties>::metadata_compression_codec(&self) -> iceberg::Result<iceberg::compression::CompressionCodec>
pub fn iceberg::spec::TableProperties<'properties>::min_snapshots_to_keep(&self) -> iceberg::Result<usize>
pub fn iceberg::spec::TableProperties<'properties>::new(properties: &'properties std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>) -> Self
pub fn iceberg::spec::TableProperties<'properties>::parquet_bloom_filter_column_enabled(&self) -> iceberg::Result<std::collections::hash::map::HashMap<alloc::string::String, bool>>
pub fn iceberg::spec::TableProperties<'properties>::parquet_bloom_filter_column_fpp(&self) -> iceberg::Result<std::collections::hash::map::HashMap<alloc::string::String, f64>>
pub fn iceberg::spec::TableProperties<'properties>::parquet_bloom_filter_column_ndv(&self) -> iceberg::Result<std::collections::hash::map::HashMap<alloc::string::String, u64>>
pub fn iceberg::spec::TableProperties<'properties>::parquet_compression_codec(&self) -> iceberg::Result<iceberg::compression::CompressionCodec>
pub fn iceberg::spec::TableProperties<'properties>::parquet_dict_size_bytes(&self) -> iceberg::Result<usize>
pub fn iceberg::spec::TableProperties<'properties>::parquet_page_row_limit(&self) -> iceberg::Result<usize>
Expand Down
33 changes: 33 additions & 0 deletions crates/iceberg/src/spec/table_properties.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,24 @@ pub struct TableProperties {
getter
)]
parquet_dict_size_bytes: usize,
/// Parquet bloom filter toggles, keyed by column name.
#[property(
prefix = Self::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX,
getter
)]
parquet_bloom_filter_column_enabled: HashMap<String, bool>,
/// Parquet bloom filter false-positive probabilities, keyed by column name.
#[property(
prefix = Self::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX,
getter
)]
parquet_bloom_filter_column_fpp: HashMap<String, f64>,
/// Parquet bloom filter distinct-value estimates, keyed by column name.
#[property(
prefix = Self::PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX,
getter
)]
parquet_bloom_filter_column_ndv: HashMap<String, u64>,
/// The master key id used to encrypt this table's manifest list and data
/// files. `None` if `encryption.key-id` is not set.
#[property(
Expand Down Expand Up @@ -530,6 +548,21 @@ impl TableProperties<'_> {
/// Default Parquet dictionary page size in bytes.
pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024;

/// Prefix for per-column Parquet bloom filter toggles, suffixed with the
/// column name, e.g. `write.parquet.bloom-filter-enabled.column.id`.
pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX: &'static str =
"write.parquet.bloom-filter-enabled.column.";

/// Prefix for per-column Parquet bloom filter false-positive probabilities.
pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX: &'static str =
"write.parquet.bloom-filter-fpp.column.";
/// Default false-positive probability for an enabled column.
pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT: f64 = 0.01;

/// Prefix for per-column Parquet bloom filter distinct-value estimates.
pub const PROPERTY_PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX: &'static str =
"write.parquet.bloom-filter-ndv.column.";

/// Property key for the master key id used to encrypt the table's manifest
/// list and data files as defined in <https://iceberg.apache.org/docs/nightly/encryption/>.
pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id";
Expand Down
Loading
Loading