From 8882985736f07accddb141b9971ab64c6ba8f977 Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Tue, 3 Nov 2026 16:50:46 -0500 Subject: [PATCH 1/7] fix(reader): respect Parquet timestamp units when pruning Row group, page index and bloom filter pruning read TIMESTAMP(MILLIS) values as microseconds. Scale them to microseconds, and decline to prune NANOS columns under a microsecond field. Footer metrics for written and imported files get the same conversion. --- crates/iceberg/src/arrow/reader/pipeline.rs | 6 +- crates/iceberg/src/arrow/schema.rs | 143 ++++++++++++++++-- .../expr/visitors/bloom_filter_evaluator.rs | 58 ++++++- .../src/expr/visitors/page_index_evaluator.rs | 133 ++++++++++++++-- .../visitors/row_group_metrics_evaluator.rs | 43 +++--- .../src/writer/file_writer/parquet_writer.rs | 91 +++++++++-- 6 files changed, 407 insertions(+), 67 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 1ddee493c4..94d48fd86c 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -38,11 +38,11 @@ use super::{ ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema, find_leaf_by_field_id, }; -use crate::arrow::build_partition_constant; use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; use crate::arrow::int96::coerce_int96_timestamps; use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult}; +use crate::arrow::{build_partition_constant, parquet_timestamp_unit}; use crate::encryption::StandardKeyMetadata; use crate::error::{Result, invalid_data}; use crate::expr::BoundPredicate; @@ -771,6 +771,7 @@ impl FileScanTaskReader { let physical_type = col_meta.column_type(); let type_length = col_meta.column_descr().type_length(); + let timestamp_unit = parquet_timestamp_unit(col_meta.column_descr()); match builder .get_row_group_column_bloom_filter(rg_idx, col_idx) @@ -779,7 +780,8 @@ impl FileScanTaskReader { Ok(Some(sbbf)) => { bloom_filters.insert( field_id, - ColumnBloomFilter::new(sbbf, physical_type, type_length), + ColumnBloomFilter::new(sbbf, physical_type, type_length) + .with_timestamp_unit(timestamp_unit), ); } Ok(None) => {} diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 91b8d687ad..c65737db4e 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -31,7 +31,9 @@ use arrow_schema::{ ArrowError, DataType, Field, FieldRef, Fields, Schema as ArrowSchema, TimeUnit, }; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use parquet::basic::{ConvertedType, LogicalType, TimeUnit as ParquetTimeUnit}; use parquet::file::statistics::Statistics; +use parquet::schema::types::ColumnDescriptor; use parquet_geospatial::{WkbEdges, WkbMetadata, WkbType, WkbTypeHint}; use uuid::Uuid; @@ -991,9 +993,32 @@ pub(crate) fn get_arrow_datum(datum: &Datum) -> Result Option { + match (column.logical_type_ref(), column.converted_type()) { + (Some(LogicalType::Timestamp(timestamp)), _) => Some(match timestamp.unit { + ParquetTimeUnit::MILLIS => TimeUnit::Millisecond, + ParquetTimeUnit::MICROS => TimeUnit::Microsecond, + ParquetTimeUnit::NANOS => TimeUnit::Nanosecond, + }), + (None, ConvertedType::TIMESTAMP_MILLIS) => Some(TimeUnit::Millisecond), + (None, ConvertedType::TIMESTAMP_MICROS) => Some(TimeUnit::Microsecond), + _ => None, + } +} + +pub(crate) fn parquet_timestamp_to_micros(value: i64, unit: Option) -> Option { + match unit { + Some(TimeUnit::Millisecond) => value.checked_mul(1_000), + // A truncated nanosecond max would undershoot and prune matching rows. + Some(TimeUnit::Nanosecond) => None, + _ => Some(value), + } +} + pub(crate) fn get_parquet_stat_min_as_datum( primitive_type: &PrimitiveType, stats: &Statistics, + timestamp_unit: Option, ) -> Result> { Ok(match (primitive_type, stats) { (PrimitiveType::Boolean, Statistics::Boolean(stats)) => { @@ -1015,12 +1040,14 @@ pub(crate) fn get_parquet_stat_min_as_datum( Some(Datum::time_micros(*val)?) } - (PrimitiveType::Timestamp, Statistics::Int64(stats)) => { - stats.min_opt().map(|val| Datum::timestamp_micros(*val)) - } - (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => { - stats.min_opt().map(|val| Datum::timestamptz_micros(*val)) - } + (PrimitiveType::Timestamp, Statistics::Int64(stats)) => stats + .min_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamp_micros), + (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => stats + .min_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamptz_micros), (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => { stats.min_opt().map(|val| Datum::timestamp_nanos(*val)) } @@ -1144,6 +1171,7 @@ pub(crate) fn get_parquet_stat_min_as_datum( pub(crate) fn get_parquet_stat_max_as_datum( primitive_type: &PrimitiveType, stats: &Statistics, + timestamp_unit: Option, ) -> Result> { Ok(match (primitive_type, stats) { (PrimitiveType::Boolean, Statistics::Boolean(stats)) => { @@ -1165,12 +1193,14 @@ pub(crate) fn get_parquet_stat_max_as_datum( Some(Datum::time_micros(*val)?) } - (PrimitiveType::Timestamp, Statistics::Int64(stats)) => { - stats.max_opt().map(|val| Datum::timestamp_micros(*val)) - } - (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => { - stats.max_opt().map(|val| Datum::timestamptz_micros(*val)) - } + (PrimitiveType::Timestamp, Statistics::Int64(stats)) => stats + .max_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamp_micros), + (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => stats + .max_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamptz_micros), (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => { stats.max_opt().map(|val| Datum::timestamp_nanos(*val)) } @@ -1553,6 +1583,8 @@ mod tests { use arrow_schema::extension::ExtensionType; use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; use parquet_geospatial::WkbEdges; use super::*; @@ -2665,16 +2697,95 @@ mod tests { PrimitiveType::Geography(GeographyType::default()), ] { assert_eq!( - get_parquet_stat_min_as_datum(&primitive_type, &stats).unwrap(), + get_parquet_stat_min_as_datum(&primitive_type, &stats, None).unwrap(), None ); assert_eq!( - get_parquet_stat_max_as_datum(&primitive_type, &stats).unwrap(), + get_parquet_stat_max_as_datum(&primitive_type, &stats, None).unwrap(), None ); } } + #[test] + fn test_parquet_timestamp_unit() { + let schema = SchemaDescriptor::new(Arc::new( + parse_message_type( + " + message schema { + required int64 utc (TIMESTAMP(MILLIS,true)); + required int64 local (TIMESTAMP(MILLIS,false)); + required int64 legacy (TIMESTAMP_MILLIS); + required int64 plain; + } + ", + ) + .unwrap(), + )); + + let units: Vec<_> = schema + .columns() + .iter() + .map(|column| parquet_timestamp_unit(column)) + .collect(); + + assert_eq!(units, vec![ + Some(TimeUnit::Millisecond), + Some(TimeUnit::Millisecond), + Some(TimeUnit::Millisecond), + None, + ]); + } + + #[test] + fn test_parquet_timestamp_stats_as_micros() { + let bounds = |primitive_type: PrimitiveType, min: i64, max: i64, unit: TimeUnit| { + let stats = Statistics::int64(Some(min), Some(max), None, Some(0), false); + ( + get_parquet_stat_min_as_datum(&primitive_type, &stats, Some(unit)).unwrap(), + get_parquet_stat_max_as_datum(&primitive_type, &stats, Some(unit)).unwrap(), + ) + }; + + assert_eq!( + bounds( + PrimitiveType::Timestamp, + -1_500, + 2_000, + TimeUnit::Millisecond + ), + ( + Some(Datum::timestamp_micros(-1_500_000)), + Some(Datum::timestamp_micros(2_000_000)) + ) + ); + assert_eq!( + bounds( + PrimitiveType::Timestamptz, + -1_500, + 2_000, + TimeUnit::Millisecond + ), + ( + Some(Datum::timestamptz_micros(-1_500_000)), + Some(Datum::timestamptz_micros(2_000_000)) + ) + ); + assert_eq!( + bounds(PrimitiveType::Timestamp, 0, i64::MAX, TimeUnit::Millisecond), + (Some(Datum::timestamp_micros(0)), None) + ); + assert_eq!( + bounds( + PrimitiveType::Timestamp, + -1_500, + 2_000, + TimeUnit::Nanosecond + ), + (None, None) + ); + } + #[test] fn test_type_conversion() { // test primitive type @@ -3124,8 +3235,8 @@ mod tests { scale: 2, }; Ok(( - get_parquet_stat_min_as_datum(&ty, stats)?, - get_parquet_stat_max_as_datum(&ty, stats)?, + get_parquet_stat_min_as_datum(&ty, stats, None)?, + get_parquet_stat_max_as_datum(&ty, stats, None)?, )) } diff --git a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs index 3dfc8ec4a9..357c795836 100644 --- a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs @@ -20,6 +20,7 @@ use std::collections::{HashMap, HashSet}; +use arrow_schema::TimeUnit; use fnv::FnvHashSet; use parquet::basic::Type as PhysicalType; use parquet::bloom_filter::Sbbf; @@ -43,6 +44,7 @@ pub(crate) struct ColumnBloomFilter { /// `type_length` from the file's column descriptor. Only meaningful for /// `FIXED_LEN_BYTE_ARRAY`. type_length: i32, + timestamp_unit: Option, } impl ColumnBloomFilter { @@ -51,8 +53,14 @@ impl ColumnBloomFilter { sbbf, physical_type, type_length, + timestamp_unit: None, } } + + pub(crate) fn with_timestamp_unit(mut self, timestamp_unit: Option) -> Self { + self.timestamp_unit = timestamp_unit; + self + } } pub(crate) struct BloomFilterEvaluator<'a> { @@ -251,6 +259,7 @@ fn check_in_bloom_filter(column: &ColumnBloomFilter, datum: &Datum) -> bool { sbbf, physical_type, type_length, + timestamp_unit, } = column; let physical_type = *physical_type; @@ -269,7 +278,17 @@ fn check_in_bloom_filter(column: &ColumnBloomFilter, datum: &Datum) -> bool { _ => true, }, PrimitiveLiteral::Long(v) => match physical_type { - PhysicalType::INT64 => sbbf.check(v), + PhysicalType::INT64 => match (datum.data_type(), timestamp_unit) { + ( + PrimitiveType::Timestamp | PrimitiveType::Timestamptz, + Some(TimeUnit::Millisecond), + ) => v % 1_000 == 0 && sbbf.check(&(v / 1_000)), + ( + PrimitiveType::Timestamp | PrimitiveType::Timestamptz, + Some(TimeUnit::Nanosecond), + ) => true, + _ => sbbf.check(v), + }, PhysicalType::INT32 => match i32::try_from(*v) { Ok(narrowed) => sbbf.check(&narrowed), // Too wide for an INT32 column to hold, so the value is provably @@ -513,7 +532,9 @@ impl BoundPredicateVisitor for BloomFilterEvaluator<'_> { mod tests { use std::collections::HashMap; use std::ops::Not; + use std::sync::Arc; + use arrow_schema::TimeUnit; use parquet::basic::Type as PhysicalType; use parquet::bloom_filter::Sbbf; use parquet::data_type::ByteArray; @@ -1612,4 +1633,39 @@ mod tests { let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); assert!(result, "non-representable double must not prune"); } + + #[test] + fn test_eq_probes_timestamp_in_column_unit() { + let schema = Arc::new(single_field_schema("ts", PrimitiveType::Timestamp)); + let filter = |unit, values: &[i64]| { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(v); + } + HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0) + .with_timestamp_unit(Some(unit)), + )]) + }; + let millis = filter(TimeUnit::Millisecond, &[-1_500, 2_000]); + let nanos = filter(TimeUnit::Nanosecond, &[2_000_000_000]); + + for (filters, micros, expected) in [ + (&millis, 2_000_000, true), + (&millis, 2_000_500, false), + (&millis, -1_500_000, true), + (&nanos, 2_000_000, true), + ] { + let predicate = Reference::new("ts") + .equal_to(Datum::timestamp_micros(micros)) + .bind(schema.clone(), true) + .unwrap(); + assert_eq!( + BloomFilterEvaluator::eval(&predicate, filters).unwrap(), + expected, + "ts = {micros}" + ); + } + } } diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index 4e51042336..28d1df2f5b 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; +use arrow_schema::TimeUnit; use fnv::FnvHashSet; use ordered_float::OrderedFloat; use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; @@ -26,6 +27,7 @@ use parquet::file::metadata::RowGroupMetaData; use parquet::file::page_index::column_index::ColumnIndexMetaData; use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use crate::arrow::{parquet_timestamp_to_micros, parquet_timestamp_unit}; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::decimal_utils::i128_from_be_bytes; @@ -168,16 +170,20 @@ impl<'a> PageIndexEvaluator<'a> { return self.select_all_rows(); }; - // Declared byte width of a FIXED_LEN_BYTE_ARRAY column, used to detect - // truncated page-index bounds when decoding decimals. `None` when the - // column metadata is absent or reports no fixed width. - let type_length = self + let column_descr = self .row_group_metadata .columns() .get(parquet_column_index) - .map(|column| column.column_descr().type_length()) + .map(|column| column.column_descr()); + + // Declared byte width of a FIXED_LEN_BYTE_ARRAY column, used to detect + // truncated page-index bounds when decoding decimals. `None` when the + // column metadata is absent or reports no fixed width. + let type_length = column_descr + .map(|descr| descr.type_length()) .filter(|&len| len > 0) .map(|len| len as usize); + let timestamp_unit = column_descr.and_then(parquet_timestamp_unit); let row_counts = { // Caches row count calculations for columns that appear multiple times in @@ -207,6 +213,7 @@ impl<'a> PageIndexEvaluator<'a> { predicate, field_type, type_length, + timestamp_unit, column_index, &row_counts, )? @@ -256,6 +263,7 @@ impl<'a> PageIndexEvaluator<'a> { predicate: F, field_type: &PrimitiveType, type_length: Option, + timestamp_unit: Option, column_index: &ColumnIndexMetaData, row_counts: &[usize], ) -> Result>> @@ -303,8 +311,12 @@ impl<'a> PageIndexEvaluator<'a> { .zip(row_counts.iter()) .map(|((i, (min, max)), &row_count)| { predicate( - min.map(|&val| Self::int64_bound_to_datum(field_type, val)), - max.map(|&val| Self::int64_bound_to_datum(field_type, val)), + min.and_then(|&val| { + Self::int64_bound_to_datum(field_type, timestamp_unit, val) + }), + max.and_then(|&val| { + Self::int64_bound_to_datum(field_type, timestamp_unit, val) + }), PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) @@ -505,13 +517,24 @@ impl<'a> PageIndexEvaluator<'a> { /// Converts an `INT64` page bound into a [`Datum`] according to the field's /// primitive type. - fn int64_bound_to_datum(field_type: &PrimitiveType, val: i64) -> Datum { + /// + /// Returns `None` for nanosecond bounds under a microsecond field, or when a + /// millisecond bound overflows, so that side of the page cannot be pruned. + fn int64_bound_to_datum( + field_type: &PrimitiveType, + timestamp_unit: Option, + val: i64, + ) -> Option { match field_type { - PrimitiveType::Decimal { .. } => Datum::new( + PrimitiveType::Decimal { .. } => Some(Datum::new( field_type.clone(), PrimitiveLiteral::Int128(i128::from(val)), - ), - _ => Datum::new(field_type.clone(), PrimitiveLiteral::Long(val)), + )), + PrimitiveType::Timestamp | PrimitiveType::Timestamptz => { + parquet_timestamp_to_micros(val, timestamp_unit) + .map(|val| Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))) + } + _ => Some(Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))), } } @@ -974,9 +997,9 @@ mod tests { use arrow_array::{ ArrayRef, Decimal128Array, FixedSizeBinaryArray, Float32Array, LargeBinaryArray, - RecordBatch, StringArray, + RecordBatch, StringArray, TimestampMillisecondArray, }; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelector, @@ -1292,6 +1315,45 @@ mod tests { Ok((metadata, temp_file)) } + fn create_millisecond_timestamp_parquet_file( + pages: &[i64], + ) -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + for page in pages { + let array = Arc::new(TimestampMillisecondArray::from_iter_values( + std::iter::repeat_n(*page, 1024), + )) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + /// Get the test metadata components for testing fn get_test_metadata( metadata: &ParquetMetaData, @@ -2312,6 +2374,51 @@ mod tests { Ok(()) } + #[test] + fn eval_eq_prunes_millisecond_timestamp_pages() -> Result<()> { + let (metadata, _temp_file) = + create_millisecond_timestamp_parquet_file(&[-1_500, 1_000, 2_000])?; + let (column_index, offset_index, row_group_metadata) = get_test_metadata(&metadata); + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::required( + 1, + "ts", + Type::Primitive(PrimitiveType::Timestamp), + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + for (micros, expected) in [ + (2_000_000, vec![ + RowSelector::skip(2048), + RowSelector::select(1024), + ]), + (-1_500_000, vec![ + RowSelector::select(1024), + RowSelector::skip(2048), + ]), + ] { + let filter = Reference::new("ts") + .equal_to(Datum::timestamp_micros(micros)) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &column_index, + &offset_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, expected, "ts = {micros}"); + } + + Ok(()) + } + /// Builds a decimal query [`Datum`] from an `unscaled` mantissa, matching /// the field's `scale` and `precision`. fn decimal_datum(unscaled: i128, scale: u32, precision: u32) -> Result { diff --git a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs index ea3e135959..11b76bb0ba 100644 --- a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs @@ -19,11 +19,14 @@ use std::collections::HashMap; +use arrow_schema::TimeUnit; use fnv::FnvHashSet; use parquet::file::metadata::RowGroupMetaData; use parquet::file::statistics::Statistics; -use crate::arrow::{get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum}; +use crate::arrow::{ + get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, parquet_timestamp_unit, +}; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, Schema}; @@ -80,6 +83,17 @@ impl<'a> RowGroupMetricsEvaluator<'a> { .statistics() } + fn timestamp_unit(&self, field_id: i32) -> Option { + let parquet_column_index = *self + .iceberg_field_id_to_parquet_column_index + .get(&field_id)?; + parquet_timestamp_unit( + self.row_group_metadata + .column(parquet_column_index) + .column_descr(), + ) + } + fn null_count(&self, field_id: i32) -> Option { self.stats_for_field_id(field_id)?.null_count_opt() } @@ -140,7 +154,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { return Ok(None); }; - get_parquet_stat_min_as_datum(&primitive_type, stats) + get_parquet_stat_min_as_datum(&primitive_type, stats, self.timestamp_unit(field_id)) } fn max_value(&self, field_id: i32) -> Result> { @@ -148,7 +162,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { return Ok(None); }; - get_parquet_stat_max_as_datum(&primitive_type, stats) + get_parquet_stat_max_as_datum(&primitive_type, stats, self.timestamp_unit(field_id)) } fn visit_inequality( @@ -310,11 +324,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_CANT_MATCH; } - let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { - return ROW_GROUP_MIGHT_MATCH; - }; - - if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { + if let Some(lower_bound) = self.min_value(field_id)? { if lower_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -324,7 +334,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { + if let Some(upper_bound) = self.max_value(field_id)? { if upper_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -368,11 +378,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { )); }; - let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { - return ROW_GROUP_MIGHT_MATCH; - }; - - if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { + if let Some(lower_bound) = self.min_value(field_id)? { let PrimitiveLiteral::String(lower_bound) = lower_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -390,7 +396,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { + if let Some(upper_bound) = self.max_value(field_id)? { let PrimitiveLiteral::String(upper_bound) = upper_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -497,11 +503,8 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_MIGHT_MATCH; } - let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { - return ROW_GROUP_MIGHT_MATCH; - }; - let lower_bound = get_parquet_stat_min_as_datum(&primitive_type, stats)?; - let upper_bound = get_parquet_stat_max_as_datum(&primitive_type, stats)?; + let lower_bound = self.min_value(field_id)?; + let upper_bound = self.max_value(field_id)?; // A NaN bound is unreliable on that side only. Drop it to unbounded // so a valid bound can still prune. See InclusiveMetricsEvaluator. diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 1470b310b9..d227dd5254 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::Arc; -use arrow_schema::SchemaRef as ArrowSchemaRef; +use arrow_schema::{SchemaRef as ArrowSchemaRef, TimeUnit}; use bytes::Bytes; use futures::future::BoxFuture; use itertools::Itertools; @@ -36,7 +36,7 @@ use parquet::file::statistics::Statistics; use super::{FileWriter, FileWriterBuilder}; use crate::arrow::{ ArrowFileReader, DEFAULT_MAP_FIELD_NAME, FieldMatchMode, NanValueCountVisitor, - get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, + get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, parquet_timestamp_unit, }; use crate::compression::CompressionCodec; use crate::encryption::{EncryptionManager, StandardKeyMetadata}; @@ -354,7 +354,12 @@ impl MinMaxColAggregator { } /// Update statistics - fn update(&mut self, field_id: i32, value: &Statistics) -> Result<()> { + fn update( + &mut self, + field_id: i32, + value: &Statistics, + timestamp_unit: Option, + ) -> Result<()> { let Some(ty) = self .schema .field_by_id(field_id) @@ -372,7 +377,7 @@ impl MinMaxColAggregator { }; if value.min_is_exact() { - let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value)? else { + let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value, timestamp_unit)? else { return Err(Error::new( ErrorKind::Unexpected, format!("Statistics {value} is not match with field type {ty}."), @@ -386,7 +391,7 @@ impl MinMaxColAggregator { } if value.max_is_exact() { - let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value)? else { + let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value, timestamp_unit)? else { return Err(Error::new( ErrorKind::Unexpected, format!("Statistics {value} is not match with field type {ty}."), @@ -485,7 +490,11 @@ impl ParquetWriter { *per_col_null_val_num.entry(field_id).or_insert(0) += null_count; } - min_max_agg.update(field_id, statistics)?; + min_max_agg.update( + field_id, + statistics, + parquet_timestamp_unit(column_chunk_metadata.column_descr()), + )?; } } } @@ -757,13 +766,13 @@ mod tests { use arrow_array::types::{Float32Type, Int64Type}; use arrow_array::{ Array, ArrayRef, BooleanArray, Decimal128Array, Float32Array, Float64Array, Int32Array, - Int64Array, ListArray, MapArray, RecordBatch, StructArray, + Int64Array, ListArray, MapArray, RecordBatch, StructArray, TimestampMillisecondArray, }; use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef}; use arrow_select::concat::concat_batches; use futures::TryStreamExt; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use parquet::file::statistics::ValueStatistics; use parquet::schema::types::ColumnPath; @@ -2592,16 +2601,16 @@ mod tests { let create_statistics = |min, max| Statistics::Int32(ValueStatistics::new(min, max, None, None, false)); min_max_agg - .update(0, &create_statistics(None, Some(42))) + .update(0, &create_statistics(None, Some(42)), None) .unwrap(); min_max_agg - .update(0, &create_statistics(Some(0), Some(i32::MAX))) + .update(0, &create_statistics(Some(0), Some(i32::MAX)), None) .unwrap(); min_max_agg - .update(0, &create_statistics(Some(i32::MIN), None)) + .update(0, &create_statistics(Some(i32::MIN), None), None) .unwrap(); min_max_agg - .update(0, &create_statistics(None, None)) + .update(0, &create_statistics(None, None), None) .unwrap(); let (lower_bounds, upper_bounds) = min_max_agg.produce(); @@ -2629,13 +2638,13 @@ mod tests { |min, max| Statistics::Double(ValueStatistics::new(min, max, None, None, false)); min_max_agg - .update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN))) + .update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN)), None) .unwrap(); min_max_agg - .update(0, &float_stats(Some(1.5), Some(2.5))) + .update(0, &float_stats(Some(1.5), Some(2.5)), None) .unwrap(); min_max_agg - .update(1, &double_stats(Some(f64::NAN), Some(f64::NAN))) + .update(1, &double_stats(Some(f64::NAN), Some(f64::NAN)), None) .unwrap(); let (lower_bounds, upper_bounds) = min_max_agg.produce(); @@ -2644,6 +2653,58 @@ mod tests { assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))])); } + #[test] + fn test_parquet_to_data_file_scales_millisecond_timestamp_bounds() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + TimestampMillisecondArray::from(vec![-1_500, 2_000]), + )]) + .unwrap(); + let mut buf = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut buf, arrow_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let metadata = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(buf)) + .unwrap() + .metadata() + .clone(); + + let data_file = ParquetWriter::parquet_to_data_file_builder( + schema, + metadata, + 0, + "ts.parquet".to_string(), + HashMap::new(), + None, + ) + .unwrap() + .partition_spec_id(0) + .build() + .unwrap(); + + assert_eq!( + data_file.lower_bounds(), + &HashMap::from([(1, Datum::timestamp_micros(-1_500_000))]) + ); + assert_eq!( + data_file.upper_bounds(), + &HashMap::from([(1, Datum::timestamp_micros(2_000_000))]) + ); + } + fn cdc_test_schema() -> SchemaRef { Arc::new( Schema::builder() From 7b9ca6bf102f4710f4d0606f9c88073098a76a22 Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Tue, 3 Nov 2026 17:32:11 -0500 Subject: [PATCH 2/7] fix(reader): compare millisecond timestamp columns at microsecond precision The row filter cast microsecond literals down to milliseconds, which truncated them. Widen millisecond columns to microseconds before comparing instead. --- .../src/arrow/reader/predicate_visitor.rs | 125 ++++++++++++++++-- 1 file changed, 114 insertions(+), 11 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..f05d238466 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -27,9 +27,9 @@ use arrow_array::cast::AsArray; use arrow_array::types::{Float32Type, Float64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; use arrow_buffer::BooleanBuffer; -use arrow_cast::cast::cast; +use arrow_cast::cast::{CastOptions, cast, cast_with_options}; use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; -use arrow_schema::{ArrowError, DataType}; +use arrow_schema::{ArrowError, DataType, TimeUnit}; use arrow_string::like::starts_with; use fnv::FnvHashSet; use parquet::schema::types::SchemaDescriptor; @@ -278,6 +278,27 @@ fn project_column( } } +/// Like [`project_column`], but widens millisecond timestamps to microseconds so +/// comparisons with microsecond literals keep their precision. +fn project_comparison_column( + batch: &RecordBatch, + column_idx: usize, +) -> std::result::Result { + let column = project_column(batch, column_idx)?; + + match column.data_type() { + DataType::Timestamp(TimeUnit::Millisecond, tz) => cast_with_options( + &column, + &DataType::Timestamp(TimeUnit::Microsecond, tz.clone()), + &CastOptions { + safe: false, + ..Default::default() + }, + ), + _ => Ok(column), + } +} + fn compute_is_nan(array: &ArrayRef) -> std::result::Result { // Compute NaN over the contiguous values slice, then fold the null bitmap // in with a single bitwise AND so that null slots become false. @@ -427,7 +448,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; lt(&left, literal.as_ref()) })) @@ -447,7 +468,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; lt_eq(&left, literal.as_ref()) })) @@ -467,7 +488,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; gt(&left, literal.as_ref()) })) @@ -487,7 +508,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; gt_eq(&left, literal.as_ref()) })) @@ -507,7 +528,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; eq(&left, literal.as_ref()) })) @@ -527,7 +548,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; neq(&left, literal.as_ref()) })) @@ -592,7 +613,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { Ok(Box::new(move |batch| { // update this if arrow ever adds a native is_in kernel - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let mut acc = constant_bool_array(false, batch.num_rows()); for literal in &literals { @@ -622,7 +643,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { Ok(Box::new(move |batch| { // update this if arrow ever adds a native not_in kernel - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let mut acc = constant_bool_array(true, batch.num_rows()); for literal in &literals { let literal = try_cast_literal(literal, left.data_type())?; @@ -672,7 +693,9 @@ mod tests { use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::{Bind, Predicate, Reference}; - use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + use crate::spec::{ + Datum, NestedField, PrimitiveLiteral, PrimitiveType, Schema, SchemaRef, Type, + }; fn table_schema_simple() -> SchemaRef { Arc::new( @@ -836,4 +859,84 @@ mod tests { ); assert!(!result.is_null(2)); } + + #[test] + fn test_predicate_converter_compares_millisecond_timestamps_as_micros() { + use arrow_array::TimestampMillisecondArray; + + for (primitive_type, tz) in [ + (PrimitiveType::Timestamp, None), + (PrimitiveType::Timestamptz, Some("UTC")), + ] { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(4, "ts", Type::Primitive(primitive_type.clone())) + .into(), + ]) + .build() + .unwrap(), + ); + let column = + TimestampMillisecondArray::from(vec![-1_500, 1_000, 2_000]).with_timezone_opt(tz); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + column.data_type().clone(), + false, + )])), + vec![Arc::new(column)], + ) + .unwrap(); + let micros = |v| Datum::new(primitive_type.clone(), PrimitiveLiteral::Long(v)); + + for (predicate, expected) in [ + (Reference::new("ts").less_than(micros(1_000_500)), [ + true, true, false, + ]), + (Reference::new("ts").equal_to(micros(1_000_500)), [ + false, false, false, + ]), + (Reference::new("ts").equal_to(micros(2_000_000)), [ + false, false, true, + ]), + ] { + let result = apply_predicate_to_batch(predicate, schema.clone(), batch.clone()); + assert_eq!(result, BooleanArray::from(expected.to_vec()), "{tz:?}"); + } + } + } + + #[test] + fn test_predicate_converter_null_checks_skip_millisecond_widening() { + use arrow_array::TimestampMillisecondArray; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(4, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + ]) + .build() + .unwrap(), + ); + let column = TimestampMillisecondArray::from(vec![Some(i64::MAX), None]); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + column.data_type().clone(), + true, + )])), + vec![Arc::new(column)], + ) + .unwrap(); + + for (predicate, expected) in [ + (Reference::new("ts").is_null(), [false, true]), + (Reference::new("ts").is_not_null(), [true, false]), + ] { + let result = apply_predicate_to_batch(predicate, schema.clone(), batch.clone()); + assert_eq!(result, BooleanArray::from(expected.to_vec())); + } + } } From b0b6c79e00b2a79a59a0388a108ddd7e74d4e2f7 Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Tue, 3 Nov 2026 17:32:35 -0500 Subject: [PATCH 3/7] fix(reader): read Parquet millisecond timestamps Map Arrow millisecond timestamps to Iceberg timestamp and timestamptz so projection widens them to microseconds. Promotion casts now fail on overflow instead of returning nulls. --- crates/iceberg/src/arrow/reader/row_filter.rs | 130 +++++++++++++++++- .../src/arrow/record_batch_transformer.rs | 22 ++- crates/iceberg/src/arrow/schema.rs | 7 +- 3 files changed, 149 insertions(+), 10 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 331e7f9656..ac96ae13a1 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -228,8 +228,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, - RecordBatch, StringArray, + Array, ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, + RecordBatch, StringArray, TimestampMillisecondArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -247,7 +247,8 @@ mod tests { use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + DataContentType, DataFileFormat, Datum, NestedField, PrimitiveLiteral, PrimitiveType, + Schema, SchemaRef, Type, }; async fn test_perform_read( @@ -1999,4 +2000,127 @@ mod tests { .await; assert_eq!(rows(&on), 1); } + + async fn read_id_and_ts( + reader: ArrowReader, + path: &str, + schema: SchemaRef, + predicate: &Predicate, + project_ts: bool, + ) -> Vec<(i32, Option)> { + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(if project_ts { vec![1, 2] } else { vec![1] }) + .with_predicate(Some(predicate.clone().bind(schema, true).unwrap())) + .with_case_sensitive(true) + .build() + .unwrap(); + let batches: Vec = reader + .read(Box::pin(futures::stream::iter(vec![Ok(task)]))) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + let mut rows = vec![]; + for batch in &batches { + let ids = batch + .column(0) + .as_primitive::(); + let ts = project_ts.then(|| { + batch + .column(1) + .as_primitive::() + }); + for row in 0..batch.num_rows() { + rows.push((ids.value(row), ts.map(|ts| ts.value(row)))); + } + } + rows + } + + #[tokio::test] + async fn test_millisecond_timestamp_column_in_every_pruning_mode() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/ts_millis.parquet", tmp.path().to_str().unwrap()); + + for (primitive_type, tz) in [ + (PrimitiveType::Timestamp, None), + (PrimitiveType::Timestamptz, Some("UTC")), + ] { + let ts = + TimestampMillisecondArray::from(vec![-1_500, 1_000, 2_000]).with_timezone_opt(tz); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, 1), + field_with_id("ts", ts.data_type().clone(), 2), + ])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![0, 1, 2])), + Arc::new(ts), + ]) + .unwrap(); + write_row_groups(&path, arrow_schema, vec![batch], true); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "ts", Type::Primitive(primitive_type.clone())) + .into(), + ]) + .build() + .unwrap(), + ); + let micros = |v| Datum::new(primitive_type.clone(), PrimitiveLiteral::Long(v)); + let cases = [ + (Reference::new("ts").equal_to(micros(2_000_000)), vec![( + 2, 2_000_000, + )]), + (Reference::new("ts").less_than(micros(1_000_500)), vec![ + (0, -1_500_000), + (1, 1_000_000), + ]), + (Reference::new("ts").equal_to(micros(1_000_500)), vec![]), + ]; + + for (mode, row_group_filtering, row_selection, bloom_filter) in [ + ("default", true, false, false), + ("page index only", false, true, false), + ("bloom filter only", false, false, true), + ("all off", false, false, false), + ] { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_row_group_filtering_enabled(row_group_filtering) + .with_row_selection_enabled(row_selection) + .with_bloom_filter_enabled(bloom_filter) + .build(); + for (predicate, expected) in &cases { + for project_ts in [false, true] { + let rows = read_id_and_ts( + reader.clone(), + &path, + schema.clone(), + predicate, + project_ts, + ) + .await; + let expected: Vec<_> = expected + .iter() + .map(|&(id, ts)| (id, project_ts.then_some(ts))) + .collect(); + assert_eq!( + rows, expected, + "{primitive_type} {mode} {predicate} project_ts={project_ts}" + ); + } + } + } + } + } } diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 108dd6ecc2..be1e3ed40a 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -24,7 +24,7 @@ use arrow_array::{ Array as ArrowArray, ArrayRef, Int32Array, Int64Array, ListArray, MapArray, RecordBatch, RecordBatchOptions, RunArray, StructArray, }; -use arrow_cast::cast; +use arrow_cast::{CastOptions, cast, cast_with_options}; use arrow_schema::{ DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SchemaRef, @@ -397,7 +397,14 @@ impl PromotePlan { fn apply(&self, array: &ArrayRef) -> Result { match self { PromotePlan::PassThrough => Ok(Arc::clone(array)), - PromotePlan::Cast(target_type) => Ok(cast(array.as_ref(), target_type)?), + PromotePlan::Cast(target_type) => Ok(cast_with_options( + array.as_ref(), + target_type, + &CastOptions { + safe: false, + ..Default::default() + }, + )?), PromotePlan::Struct { fields, children } => { let source = array .as_struct_opt() @@ -1427,7 +1434,8 @@ mod test { use arrow_array::types::Int32Type; use arrow_array::{ Array, ArrayRef, Date32Array, Float32Array, Float64Array, Int32Array, Int64Array, - LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray, new_null_array, + LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray, + TimestampMillisecondArray, new_null_array, }; use arrow_buffer::{NullBuffer, OffsetBuffer}; use arrow_cast::cast; @@ -1500,6 +1508,14 @@ mod test { Ok(transformer.process_record_batch(batch)?.column(1).clone()) } + #[test] + fn promote_millisecond_timestamp_overflow_errors() { + let column = NestedField::optional(2, "ts", Type::Primitive(PrimitiveType::Timestamp)); + let file_column = Arc::new(TimestampMillisecondArray::from(vec![i64::MAX])) as ArrayRef; + + assert!(transform_top_level(column, file_column).is_err()); + } + #[test] fn promote_struct_fills_added_fields_by_id() { let source = Arc::new(StructArray::new( diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index c65737db4e..cea3bf0a4f 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -642,15 +642,14 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Time64(unit) if unit == &TimeUnit::Microsecond => { Ok(Type::Primitive(PrimitiveType::Time)) } - DataType::Timestamp(unit, None) if unit == &TimeUnit::Microsecond => { + DataType::Timestamp(TimeUnit::Millisecond | TimeUnit::Microsecond, None) => { Ok(Type::Primitive(PrimitiveType::Timestamp)) } DataType::Timestamp(unit, None) if unit == &TimeUnit::Nanosecond => { Ok(Type::Primitive(PrimitiveType::TimestampNs)) } - DataType::Timestamp(unit, Some(zone)) - if unit == &TimeUnit::Microsecond - && (zone.as_ref() == "UTC" || zone.as_ref() == "+00:00") => + DataType::Timestamp(TimeUnit::Millisecond | TimeUnit::Microsecond, Some(zone)) + if zone.as_ref() == "UTC" || zone.as_ref() == "+00:00" => { Ok(Type::Primitive(PrimitiveType::Timestamptz)) } From ef518af06a79ac19af1c611a6e787756ee6feb59 Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Wed, 4 Nov 2026 08:08:35 -0500 Subject: [PATCH 4/7] fix(reader): keep rejecting millisecond columns under nanosecond fields Projecting a millisecond column under a timestamp_ns or timestamptz_ns field failed on main. Keep that error instead of reading the column as nulls. --- crates/iceberg/src/arrow/reader/projection.rs | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 3474b21624..5d40d6c29f 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -24,7 +24,7 @@ use std::str::FromStr; use std::sync::Arc; use arrow_schema::{ - DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, + DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, TimeUnit, }; use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask}; use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; @@ -189,6 +189,22 @@ impl ArrowReader { }), arrow_schema.metadata().clone(), ); + // A millisecond column under a nanosecond field would otherwise read as nulls. + for field in projected_arrow_schema.flattened_fields() { + if let DataType::Timestamp(TimeUnit::Millisecond, _) = field.data_type() + && let Some(field_id) = parquet_field_id(field) + && let Some(Type::Primitive( + PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs, + )) = iceberg_schema_of_task + .field_by_id(field_id) + .map(|field| field.field_type.as_ref()) + { + return Err(invalid_data!( + "Unsupported Arrow data type: {}", + field.data_type() + )); + } + } let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; fields.filter_leaves(|idx, field| { @@ -718,6 +734,60 @@ message schema { assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0])); } + #[test] + fn test_arrow_projection_mask_rejects_millisecond_column_under_nanosecond_field() { + let message_type = " +message schema { + optional int64 ts (TIMESTAMP(MILLIS,false)) = 1; +} + "; + let parquet_type = parse_message_type(message_type).expect("should parse schema"); + let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); + + for (field_type, tz) in [ + (PrimitiveType::TimestampNs, None), + (PrimitiveType::TimestamptzNs, Some("UTC".into())), + (PrimitiveType::Timestamp, None), + ] { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(field_type.clone())).into(), + ]) + .build() + .unwrap(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Millisecond, tz), true) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let result = ArrowReader::get_arrow_projection_mask( + &[1], + &schema, + &parquet_schema, + &arrow_schema, + false, + ); + + if field_type == PrimitiveType::Timestamp { + assert_eq!( + result.unwrap(), + ProjectionMask::leaves(&parquet_schema, vec![0]) + ); + } else { + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{field_type}"); + assert!( + err.to_string() + .contains("Unsupported Arrow data type: Timestamp(ms"), + "{err}" + ); + } + } + } + #[test] fn test_arrow_projection_mask_variant_is_unsupported() { // Reading variant columns is not supported yet: projecting one (top-level or From 770dea952671609b616818553c907fb2e920d76c Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Sun, 15 Nov 2026 10:51:09 -0500 Subject: [PATCH 5/7] fix(reader): reject millisecond columns that do not match the table type A millisecond column whose Iceberg type differs from the table field, such as a naive column under timestamptz or a UTC column under timestamp, was dropped from the projection and read as nulls. Reject it with an error naming the column and both types. This also covers the nanosecond fields the previous check handled. --- crates/iceberg/src/arrow/reader/projection.rs | 146 +++++++++++------- 1 file changed, 90 insertions(+), 56 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index d8774342e3..675035cbc5 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -214,20 +214,22 @@ impl ArrowReader { }), arrow_schema.metadata().clone(), ); - // A millisecond column under a nanosecond field would otherwise read as nulls. + // A millisecond column whose Iceberg type differs from the table's would otherwise be + // dropped and read as nulls. for field in projected_arrow_schema.flattened_fields() { if let DataType::Timestamp(TimeUnit::Millisecond, _) = field.data_type() && let Some(field_id) = parquet_field_id(field) - && let Some(Type::Primitive( - PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs, - )) = iceberg_schema_of_task - .field_by_id(field_id) - .map(|field| field.field_type.as_ref()) + && let Some(table_field) = iceberg_schema_of_task.field_by_id(field_id) { - return Err(invalid_data!( - "Unsupported Arrow data type: {}", - field.data_type() - )); + let file_type = arrow_type_to_type(field.data_type())?; + if file_type != *table_field.field_type { + return Err(invalid_data!( + "Cannot read column {} (field id {field_id}) as {}: the file stores {} which reads as {file_type}", + table_field.name, + table_field.field_type, + field.data_type() + )); + } } } let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; @@ -1027,6 +1029,7 @@ mod tests { use parquet::file::properties::WriterProperties; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; + use rstest::rstest; use tempfile::TempDir; use crate::arrow::{ArrowReader, ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions}; @@ -1159,55 +1162,86 @@ message schema { assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0])); } - #[test] - fn test_arrow_projection_mask_rejects_millisecond_column_under_nanosecond_field() { - let message_type = " -message schema { - optional int64 ts (TIMESTAMP(MILLIS,false)) = 1; -} - "; - let parquet_type = parse_message_type(message_type).expect("should parse schema"); - let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); - - for (field_type, tz) in [ - (PrimitiveType::TimestampNs, None), - (PrimitiveType::TimestamptzNs, Some("UTC".into())), - (PrimitiveType::Timestamp, None), - ] { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::optional(1, "ts", Type::Primitive(field_type.clone())).into(), - ]) - .build() - .unwrap(); - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("ts", DataType::Timestamp(TimeUnit::Millisecond, tz), true) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - ])); + #[rstest] + #[case::naive_under_timestamp("TIMESTAMP(MILLIS,false)", PrimitiveType::Timestamp, None)] + #[case::utc_under_timestamptz("TIMESTAMP(MILLIS,true)", PrimitiveType::Timestamptz, None)] + #[case::legacy_under_timestamptz("TIMESTAMP_MILLIS", PrimitiveType::Timestamptz, None)] + #[case::naive_under_timestamptz( + "TIMESTAMP(MILLIS,false)", + PrimitiveType::Timestamptz, + Some("Timestamp(ms) which reads as timestamp") + )] + #[case::utc_under_timestamp( + "TIMESTAMP(MILLIS,true)", + PrimitiveType::Timestamp, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + #[case::legacy_under_timestamp( + "TIMESTAMP_MILLIS", + PrimitiveType::Timestamp, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + #[case::naive_under_timestamp_ns( + "TIMESTAMP(MILLIS,false)", + PrimitiveType::TimestampNs, + Some("Timestamp(ms) which reads as timestamp") + )] + #[case::naive_under_timestamptz_ns( + "TIMESTAMP(MILLIS,false)", + PrimitiveType::TimestamptzNs, + Some("Timestamp(ms) which reads as timestamp") + )] + #[case::utc_under_timestamp_ns( + "TIMESTAMP(MILLIS,true)", + PrimitiveType::TimestampNs, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + #[case::utc_under_timestamptz_ns( + "TIMESTAMP(MILLIS,true)", + PrimitiveType::TimestamptzNs, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + fn test_arrow_projection_mask_millisecond_column_must_match_table_type( + #[case] declared: &str, + #[case] table_type: PrimitiveType, + #[case] rejected: Option<&str>, + ) { + let parquet_schema = SchemaDescriptor::new(Arc::new( + parse_message_type(&format!( + "message schema {{ required int32 id = 1; optional int64 ts ({declared}) = 2; }}" + )) + .unwrap(), + )); + let arrow_schema = Arc::new(parquet_to_arrow_schema(&parquet_schema, None).unwrap()); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "ts", Type::Primitive(table_type.clone())).into(), + ]) + .build() + .unwrap(); - let result = ArrowReader::get_arrow_projection_mask( - &[1], - &schema, - &parquet_schema, - &arrow_schema, - false, - ); + let result = ArrowReader::get_arrow_projection_mask( + &[1, 2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ); - if field_type == PrimitiveType::Timestamp { - assert_eq!( - result.unwrap(), - ProjectionMask::leaves(&parquet_schema, vec![0]) - ); - } else { + match rejected { + None => assert_eq!( + result.unwrap(), + ProjectionMask::leaves(&parquet_schema, vec![0, 1]) + ), + Some(file_type) => { let err = result.unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid, "{field_type}"); - assert!( - err.to_string() - .contains("Unsupported Arrow data type: Timestamp(ms"), - "{err}" + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.to_string(), + format!( + "DataInvalid => Cannot read column ts (field id 2) as {table_type}: the file stores {file_type}" + ) ); } } From 9b7cb5f93530b0e4ed8919247fd0a5f81af98fb1 Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Sun, 15 Nov 2026 10:59:15 -0500 Subject: [PATCH 6/7] fix(reader): fail promotion casts with DataInvalid A promotion cast that overflows, such as a date past the last timestamp_ns or a millisecond value beyond the microsecond range, returned an Unexpected error with the message Arrow Schema Error. Report it as DataInvalid and name the source and target types. Cover the date case from the Iceberg spec, which requires a runtime failure for values outside the promoted type's range, and note the change in the changelog. --- CHANGELOG.md | 1 + .../src/arrow/record_batch_transformer.rs | 30 ++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab34c96d1..8a6195f27b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * `iceberg_catalog_loader::BoxedCatalogBuilder` has a new required method, `with_runtime`. Types implementing `CatalogBuilder` get it through the blanket impl; types implementing `BoxedCatalogBuilder` directly (for example, a wrapper around another `Box`) must implement it, typically by forwarding the runtime to the builder they wrap. * `Transaction::commit` now returns `ErrorKind::PreconditionFailed` for read-only tables (`Table::readonly()`, e.g. from `StaticTable::into_table` or `TableBuilder::readonly(true)`), including empty transactions, which previously returned the table unchanged. Java treats an empty transaction on a static table as a no-op. * `ArrowReader` reads nested struct fields as null for Parquet files that have neither field ids nor a name mapping, as iceberg-java does. Earlier versions returned their stored values by position. Tables migrated without field ids need a name mapping (`schema.name-mapping.default`) that covers nested fields to read them. +* `ArrowReader` now fails with `ErrorKind::DataInvalid` instead of returning null when a promoted column holds a value that does not fit the target type, for example a `date` promoted to `timestamp_ns` beyond the year 2262 or a millisecond timestamp beyond about ±292,000 years. ## [v0.10.1] - 2026-07-28 diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index cc16559560..46ec513e7b 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -454,14 +454,19 @@ impl PromotePlan { fn apply(&self, array: &ArrayRef) -> Result { match self { PromotePlan::PassThrough => Ok(Arc::clone(array)), - PromotePlan::Cast(target_type) => Ok(cast_with_options( - array.as_ref(), - target_type, - &CastOptions { + PromotePlan::Cast(target_type) => { + cast_with_options(array.as_ref(), target_type, &CastOptions { safe: false, ..Default::default() - }, - )?), + }) + .map_err(|e| { + invalid_data!( + "cannot promote a value from {} to {target_type}", + array.data_type() + ) + .with_source(e) + }) + } PromotePlan::Struct { fields, children } => { let source = array .as_struct_opt() @@ -1603,7 +1608,18 @@ mod test { let column = NestedField::optional(2, "ts", Type::Primitive(PrimitiveType::Timestamp)); let file_column = Arc::new(TimestampMillisecondArray::from(vec![i64::MAX])) as ArrayRef; - assert!(transform_top_level(column, file_column).is_err()); + let err = transform_top_level(column, file_column).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn promote_date_beyond_timestamp_ns_range_errors() { + // Day 200000 is in the year 2517, after the last `timestamp_ns`. + let column = NestedField::optional(2, "d", Type::Primitive(PrimitiveType::TimestampNs)); + let file_column = Arc::new(Date32Array::from(vec![18000, 200000])) as ArrayRef; + + let err = transform_top_level(column, file_column).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); } #[test] From 79a1cfab6c7c6fda4009eb7ff17d9e2268a4edc3 Mon Sep 17 00:00:00 2001 From: Harrison Crosse Date: Sun, 15 Nov 2026 10:59:51 -0500 Subject: [PATCH 7/7] fix(reader): cover more timestamp units in tests Probe the millisecond bloom filter with a whole millisecond value that the filter does not hold, so an evaluator that accepts every whole millisecond fails. Map microsecond, nanosecond and legacy microsecond Parquet columns in the timestamp unit test. --- crates/iceberg/src/arrow/schema.rs | 6 ++++++ crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 3fb6674299..0a1c9cfbc7 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -2826,6 +2826,9 @@ mod tests { required int64 utc (TIMESTAMP(MILLIS,true)); required int64 local (TIMESTAMP(MILLIS,false)); required int64 legacy (TIMESTAMP_MILLIS); + required int64 micros (TIMESTAMP(MICROS,true)); + required int64 nanos (TIMESTAMP(NANOS,false)); + required int64 legacy_micros (TIMESTAMP_MICROS); required int64 plain; } ", @@ -2843,6 +2846,9 @@ mod tests { Some(TimeUnit::Millisecond), Some(TimeUnit::Millisecond), Some(TimeUnit::Millisecond), + Some(TimeUnit::Microsecond), + Some(TimeUnit::Nanosecond), + Some(TimeUnit::Microsecond), None, ]); } diff --git a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs index 357c795836..9228f169c6 100644 --- a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs @@ -1655,6 +1655,7 @@ mod tests { (&millis, 2_000_000, true), (&millis, 2_000_500, false), (&millis, -1_500_000, true), + (&millis, 3_000_000, false), (&nanos, 2_000_000, true), ] { let predicate = Reference::new("ts")