diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab34c96d1..8a6195f27b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * `iceberg_catalog_loader::BoxedCatalogBuilder` has a new required method, `with_runtime`. Types implementing `CatalogBuilder` get it through the blanket impl; types implementing `BoxedCatalogBuilder` directly (for example, a wrapper around another `Box`) must implement it, typically by forwarding the runtime to the builder they wrap. * `Transaction::commit` now returns `ErrorKind::PreconditionFailed` for read-only tables (`Table::readonly()`, e.g. from `StaticTable::into_table` or `TableBuilder::readonly(true)`), including empty transactions, which previously returned the table unchanged. Java treats an empty transaction on a static table as a no-op. * `ArrowReader` reads nested struct fields as null for Parquet files that have neither field ids nor a name mapping, as iceberg-java does. Earlier versions returned their stored values by position. Tables migrated without field ids need a name mapping (`schema.name-mapping.default`) that covers nested fields to read them. +* `ArrowReader` now fails with `ErrorKind::DataInvalid` instead of returning null when a promoted column holds a value that does not fit the target type, for example a `date` promoted to `timestamp_ns` beyond the year 2262 or a millisecond timestamp beyond about ±292,000 years. ## [v0.10.1] - 2026-07-28 diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 4e9a0756e4..8e26a3de57 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -38,11 +38,11 @@ use super::{ ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema, coerce_fixed_size_lists, find_leaf_by_field_id, }; -use crate::arrow::build_partition_constant; use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; use crate::arrow::int96::coerce_int96_timestamps; use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult}; +use crate::arrow::{build_partition_constant, parquet_timestamp_unit}; use crate::encryption::StandardKeyMetadata; use crate::error::{Result, invalid_data}; use crate::expr::BoundPredicate; @@ -784,6 +784,7 @@ impl FileScanTaskReader { let physical_type = col_meta.column_type(); let type_length = col_meta.column_descr().type_length(); + let timestamp_unit = parquet_timestamp_unit(col_meta.column_descr()); match builder .get_row_group_column_bloom_filter(rg_idx, col_idx) @@ -792,7 +793,8 @@ impl FileScanTaskReader { Ok(Some(sbbf)) => { bloom_filters.insert( field_id, - ColumnBloomFilter::new(sbbf, physical_type, type_length), + ColumnBloomFilter::new(sbbf, physical_type, type_length) + .with_timestamp_unit(timestamp_unit), ); } Ok(None) => {} diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 264c34a9a3..f05d238466 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -27,9 +27,9 @@ use arrow_array::cast::AsArray; use arrow_array::types::{Float32Type, Float64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; use arrow_buffer::BooleanBuffer; -use arrow_cast::cast::cast; +use arrow_cast::cast::{CastOptions, cast, cast_with_options}; use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; -use arrow_schema::{ArrowError, DataType}; +use arrow_schema::{ArrowError, DataType, TimeUnit}; use arrow_string::like::starts_with; use fnv::FnvHashSet; use parquet::schema::types::SchemaDescriptor; @@ -278,6 +278,27 @@ fn project_column( } } +/// Like [`project_column`], but widens millisecond timestamps to microseconds so +/// comparisons with microsecond literals keep their precision. +fn project_comparison_column( + batch: &RecordBatch, + column_idx: usize, +) -> std::result::Result { + let column = project_column(batch, column_idx)?; + + match column.data_type() { + DataType::Timestamp(TimeUnit::Millisecond, tz) => cast_with_options( + &column, + &DataType::Timestamp(TimeUnit::Microsecond, tz.clone()), + &CastOptions { + safe: false, + ..Default::default() + }, + ), + _ => Ok(column), + } +} + fn compute_is_nan(array: &ArrayRef) -> std::result::Result { // Compute NaN over the contiguous values slice, then fold the null bitmap // in with a single bitwise AND so that null slots become false. @@ -427,7 +448,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; lt(&left, literal.as_ref()) })) @@ -447,7 +468,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; lt_eq(&left, literal.as_ref()) })) @@ -467,7 +488,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; gt(&left, literal.as_ref()) })) @@ -487,7 +508,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; gt_eq(&left, literal.as_ref()) })) @@ -507,7 +528,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; eq(&left, literal.as_ref()) })) @@ -527,7 +548,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { let literal = get_arrow_datum(literal)?; Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let literal = try_cast_literal(&literal, left.data_type())?; neq(&left, literal.as_ref()) })) @@ -592,7 +613,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { Ok(Box::new(move |batch| { // update this if arrow ever adds a native is_in kernel - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let mut acc = constant_bool_array(false, batch.num_rows()); for literal in &literals { @@ -622,7 +643,7 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { Ok(Box::new(move |batch| { // update this if arrow ever adds a native not_in kernel - let left = project_column(&batch, idx)?; + let left = project_comparison_column(&batch, idx)?; let mut acc = constant_bool_array(true, batch.num_rows()); for literal in &literals { let literal = try_cast_literal(literal, left.data_type())?; @@ -672,7 +693,9 @@ mod tests { use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::{Bind, Predicate, Reference}; - use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + use crate::spec::{ + Datum, NestedField, PrimitiveLiteral, PrimitiveType, Schema, SchemaRef, Type, + }; fn table_schema_simple() -> SchemaRef { Arc::new( @@ -836,4 +859,84 @@ mod tests { ); assert!(!result.is_null(2)); } + + #[test] + fn test_predicate_converter_compares_millisecond_timestamps_as_micros() { + use arrow_array::TimestampMillisecondArray; + + for (primitive_type, tz) in [ + (PrimitiveType::Timestamp, None), + (PrimitiveType::Timestamptz, Some("UTC")), + ] { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(4, "ts", Type::Primitive(primitive_type.clone())) + .into(), + ]) + .build() + .unwrap(), + ); + let column = + TimestampMillisecondArray::from(vec![-1_500, 1_000, 2_000]).with_timezone_opt(tz); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + column.data_type().clone(), + false, + )])), + vec![Arc::new(column)], + ) + .unwrap(); + let micros = |v| Datum::new(primitive_type.clone(), PrimitiveLiteral::Long(v)); + + for (predicate, expected) in [ + (Reference::new("ts").less_than(micros(1_000_500)), [ + true, true, false, + ]), + (Reference::new("ts").equal_to(micros(1_000_500)), [ + false, false, false, + ]), + (Reference::new("ts").equal_to(micros(2_000_000)), [ + false, false, true, + ]), + ] { + let result = apply_predicate_to_batch(predicate, schema.clone(), batch.clone()); + assert_eq!(result, BooleanArray::from(expected.to_vec()), "{tz:?}"); + } + } + } + + #[test] + fn test_predicate_converter_null_checks_skip_millisecond_widening() { + use arrow_array::TimestampMillisecondArray; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(4, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + ]) + .build() + .unwrap(), + ); + let column = TimestampMillisecondArray::from(vec![Some(i64::MAX), None]); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + column.data_type().clone(), + true, + )])), + vec![Arc::new(column)], + ) + .unwrap(); + + for (predicate, expected) in [ + (Reference::new("ts").is_null(), [false, true]), + (Reference::new("ts").is_not_null(), [true, false]), + ] { + let result = apply_predicate_to_batch(predicate, schema.clone(), batch.clone()); + assert_eq!(result, BooleanArray::from(expected.to_vec())); + } + } } diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index 676d8498d7..675035cbc5 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -25,7 +25,7 @@ use std::str::FromStr; use std::sync::Arc; use arrow_schema::{ - DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, + DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, TimeUnit, }; use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask}; use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; @@ -214,6 +214,24 @@ impl ArrowReader { }), arrow_schema.metadata().clone(), ); + // A millisecond column whose Iceberg type differs from the table's would otherwise be + // dropped and read as nulls. + for field in projected_arrow_schema.flattened_fields() { + if let DataType::Timestamp(TimeUnit::Millisecond, _) = field.data_type() + && let Some(field_id) = parquet_field_id(field) + && let Some(table_field) = iceberg_schema_of_task.field_by_id(field_id) + { + let file_type = arrow_type_to_type(field.data_type())?; + if file_type != *table_field.field_type { + return Err(invalid_data!( + "Cannot read column {} (field id {field_id}) as {}: the file stores {} which reads as {file_type}", + table_field.name, + table_field.field_type, + field.data_type() + )); + } + } + } let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; tagged_fields.filter_leaves(|idx, field| { @@ -1011,6 +1029,7 @@ mod tests { use parquet::file::properties::WriterProperties; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; + use rstest::rstest; use tempfile::TempDir; use crate::arrow::{ArrowReader, ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions}; @@ -1143,6 +1162,91 @@ message schema { assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0])); } + #[rstest] + #[case::naive_under_timestamp("TIMESTAMP(MILLIS,false)", PrimitiveType::Timestamp, None)] + #[case::utc_under_timestamptz("TIMESTAMP(MILLIS,true)", PrimitiveType::Timestamptz, None)] + #[case::legacy_under_timestamptz("TIMESTAMP_MILLIS", PrimitiveType::Timestamptz, None)] + #[case::naive_under_timestamptz( + "TIMESTAMP(MILLIS,false)", + PrimitiveType::Timestamptz, + Some("Timestamp(ms) which reads as timestamp") + )] + #[case::utc_under_timestamp( + "TIMESTAMP(MILLIS,true)", + PrimitiveType::Timestamp, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + #[case::legacy_under_timestamp( + "TIMESTAMP_MILLIS", + PrimitiveType::Timestamp, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + #[case::naive_under_timestamp_ns( + "TIMESTAMP(MILLIS,false)", + PrimitiveType::TimestampNs, + Some("Timestamp(ms) which reads as timestamp") + )] + #[case::naive_under_timestamptz_ns( + "TIMESTAMP(MILLIS,false)", + PrimitiveType::TimestamptzNs, + Some("Timestamp(ms) which reads as timestamp") + )] + #[case::utc_under_timestamp_ns( + "TIMESTAMP(MILLIS,true)", + PrimitiveType::TimestampNs, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + #[case::utc_under_timestamptz_ns( + "TIMESTAMP(MILLIS,true)", + PrimitiveType::TimestamptzNs, + Some("Timestamp(ms, \"UTC\") which reads as timestamptz") + )] + fn test_arrow_projection_mask_millisecond_column_must_match_table_type( + #[case] declared: &str, + #[case] table_type: PrimitiveType, + #[case] rejected: Option<&str>, + ) { + let parquet_schema = SchemaDescriptor::new(Arc::new( + parse_message_type(&format!( + "message schema {{ required int32 id = 1; optional int64 ts ({declared}) = 2; }}" + )) + .unwrap(), + )); + let arrow_schema = Arc::new(parquet_to_arrow_schema(&parquet_schema, None).unwrap()); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "ts", Type::Primitive(table_type.clone())).into(), + ]) + .build() + .unwrap(); + + let result = ArrowReader::get_arrow_projection_mask( + &[1, 2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ); + + match rejected { + None => assert_eq!( + result.unwrap(), + ProjectionMask::leaves(&parquet_schema, vec![0, 1]) + ), + Some(file_type) => { + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.to_string(), + format!( + "DataInvalid => Cannot read column ts (field id 2) as {table_type}: the file stores {file_type}" + ) + ); + } + } + } + fn field_id_meta(id: i32) -> HashMap { HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())]) } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 5f1fe5efed..f4e80d8bca 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -226,8 +226,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, - RecordBatch, StringArray, + Array, ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, + RecordBatch, StringArray, TimestampMillisecondArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -248,7 +248,8 @@ mod tests { use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + DataContentType, DataFileFormat, Datum, NestedField, PrimitiveLiteral, PrimitiveType, + Schema, SchemaRef, Type, }; async fn test_perform_read( @@ -2067,4 +2068,131 @@ mod tests { .await; assert_eq!(rows(&on), 1); } + + async fn read_id_and_ts( + reader: ArrowReader, + path: &str, + schema: SchemaRef, + predicate: &Predicate, + project_ts: bool, + ) -> Vec<(i32, Option)> { + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(if project_ts { vec![1, 2] } else { vec![1] }) + .with_predicate(Some(predicate.clone().bind(schema, true).unwrap())) + .with_case_sensitive(true) + .build() + .unwrap(); + let batches: Vec = reader + .read(Box::pin(futures::stream::iter(vec![Ok(task)]))) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + let mut rows = vec![]; + for batch in &batches { + let ids = batch + .column(0) + .as_primitive::(); + let ts = project_ts.then(|| { + batch + .column(1) + .as_primitive::() + }); + for row in 0..batch.num_rows() { + rows.push((ids.value(row), ts.map(|ts| ts.value(row)))); + } + } + rows + } + + #[tokio::test] + async fn test_millisecond_timestamp_column_in_every_pruning_mode() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/ts_millis.parquet", tmp.path().to_str().unwrap()); + + for (primitive_type, tz) in [ + (PrimitiveType::Timestamp, None), + (PrimitiveType::Timestamptz, Some("UTC")), + ] { + let ts = + TimestampMillisecondArray::from(vec![-1_500, 1_000, 2_000]).with_timezone_opt(tz); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, 1), + field_with_id("ts", ts.data_type().clone(), 2), + ])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![0, 1, 2])), + Arc::new(ts), + ]) + .unwrap(); + write_row_groups(&path, arrow_schema, vec![batch], true); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "ts", Type::Primitive(primitive_type.clone())) + .into(), + ]) + .build() + .unwrap(), + ); + let micros = |v| Datum::new(primitive_type.clone(), PrimitiveLiteral::Long(v)); + let cases = [ + (Reference::new("ts").equal_to(micros(2_000_000)), vec![( + 2, 2_000_000, + )]), + (Reference::new("ts").less_than(micros(1_000_500)), vec![ + (0, -1_500_000), + (1, 1_000_000), + ]), + (Reference::new("ts").equal_to(micros(1_000_500)), vec![]), + ]; + + for (mode, row_group_filtering, row_selection, bloom_filter) in [ + ("default", true, false, false), + ("page index only", false, true, false), + ("bloom filter only", false, false, true), + ("all off", false, false, false), + ] { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(row_group_filtering) + .with_row_selection_enabled(row_selection) + .with_bloom_filter_enabled(bloom_filter) + .build(), + ) + .build(); + for (predicate, expected) in &cases { + for project_ts in [false, true] { + let rows = read_id_and_ts( + reader.clone(), + &path, + schema.clone(), + predicate, + project_ts, + ) + .await; + let expected: Vec<_> = expected + .iter() + .map(|&(id, ts)| (id, project_ts.then_some(ts))) + .collect(); + assert_eq!( + rows, expected, + "{primitive_type} {mode} {predicate} project_ts={project_ts}" + ); + } + } + } + } + } } diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 9cd626547e..46ec513e7b 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -24,7 +24,7 @@ use arrow_array::{ Array as ArrowArray, ArrayRef, Int32Array, Int64Array, ListArray, MapArray, RecordBatch, RecordBatchOptions, RunArray, StructArray, }; -use arrow_cast::cast; +use arrow_cast::{CastOptions, cast, cast_with_options}; use arrow_schema::{ DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SchemaRef, @@ -454,7 +454,19 @@ impl PromotePlan { fn apply(&self, array: &ArrayRef) -> Result { match self { PromotePlan::PassThrough => Ok(Arc::clone(array)), - PromotePlan::Cast(target_type) => Ok(cast(array.as_ref(), target_type)?), + PromotePlan::Cast(target_type) => { + cast_with_options(array.as_ref(), target_type, &CastOptions { + safe: false, + ..Default::default() + }) + .map_err(|e| { + invalid_data!( + "cannot promote a value from {} to {target_type}", + array.data_type() + ) + .with_source(e) + }) + } PromotePlan::Struct { fields, children } => { let source = array .as_struct_opt() @@ -1517,7 +1529,8 @@ mod test { use arrow_array::types::Int32Type; use arrow_array::{ Array, ArrayRef, Date32Array, Float32Array, Float64Array, Int32Array, Int64Array, - LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray, new_null_array, + LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray, + TimestampMillisecondArray, new_null_array, }; use arrow_buffer::{NullBuffer, OffsetBuffer}; use arrow_cast::cast; @@ -1590,6 +1603,25 @@ mod test { Ok(transformer.process_record_batch(batch)?.column(1).clone()) } + #[test] + fn promote_millisecond_timestamp_overflow_errors() { + let column = NestedField::optional(2, "ts", Type::Primitive(PrimitiveType::Timestamp)); + let file_column = Arc::new(TimestampMillisecondArray::from(vec![i64::MAX])) as ArrayRef; + + let err = transform_top_level(column, file_column).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn promote_date_beyond_timestamp_ns_range_errors() { + // Day 200000 is in the year 2517, after the last `timestamp_ns`. + let column = NestedField::optional(2, "d", Type::Primitive(PrimitiveType::TimestampNs)); + let file_column = Arc::new(Date32Array::from(vec![18000, 200000])) as ArrayRef; + + let err = transform_top_level(column, file_column).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + #[test] fn promote_struct_fills_added_fields_by_id() { let source = Arc::new(StructArray::new( diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 35d7214a08..0a1c9cfbc7 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -31,7 +31,9 @@ use arrow_schema::{ ArrowError, DataType, Field, FieldRef, Fields, Schema as ArrowSchema, TimeUnit, }; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use parquet::basic::{ConvertedType, LogicalType, TimeUnit as ParquetTimeUnit}; use parquet::file::statistics::Statistics; +use parquet::schema::types::ColumnDescriptor; use parquet_geospatial::{WkbEdges, WkbMetadata, WkbType, WkbTypeHint}; use uuid::Uuid; @@ -709,15 +711,14 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Time64(unit) if unit == &TimeUnit::Microsecond => { Ok(Type::Primitive(PrimitiveType::Time)) } - DataType::Timestamp(unit, None) if unit == &TimeUnit::Microsecond => { + DataType::Timestamp(TimeUnit::Millisecond | TimeUnit::Microsecond, None) => { Ok(Type::Primitive(PrimitiveType::Timestamp)) } DataType::Timestamp(unit, None) if unit == &TimeUnit::Nanosecond => { Ok(Type::Primitive(PrimitiveType::TimestampNs)) } - DataType::Timestamp(unit, Some(zone)) - if unit == &TimeUnit::Microsecond - && (zone.as_ref() == "UTC" || zone.as_ref() == "+00:00") => + DataType::Timestamp(TimeUnit::Millisecond | TimeUnit::Microsecond, Some(zone)) + if zone.as_ref() == "UTC" || zone.as_ref() == "+00:00" => { Ok(Type::Primitive(PrimitiveType::Timestamptz)) } @@ -1061,9 +1062,32 @@ pub(crate) fn get_arrow_datum(datum: &Datum) -> Result Option { + match (column.logical_type_ref(), column.converted_type()) { + (Some(LogicalType::Timestamp(timestamp)), _) => Some(match timestamp.unit { + ParquetTimeUnit::MILLIS => TimeUnit::Millisecond, + ParquetTimeUnit::MICROS => TimeUnit::Microsecond, + ParquetTimeUnit::NANOS => TimeUnit::Nanosecond, + }), + (None, ConvertedType::TIMESTAMP_MILLIS) => Some(TimeUnit::Millisecond), + (None, ConvertedType::TIMESTAMP_MICROS) => Some(TimeUnit::Microsecond), + _ => None, + } +} + +pub(crate) fn parquet_timestamp_to_micros(value: i64, unit: Option) -> Option { + match unit { + Some(TimeUnit::Millisecond) => value.checked_mul(1_000), + // A truncated nanosecond max would undershoot and prune matching rows. + Some(TimeUnit::Nanosecond) => None, + _ => Some(value), + } +} + pub(crate) fn get_parquet_stat_min_as_datum( primitive_type: &PrimitiveType, stats: &Statistics, + timestamp_unit: Option, ) -> Result> { Ok(match (primitive_type, stats) { (PrimitiveType::Boolean, Statistics::Boolean(stats)) => { @@ -1085,12 +1109,14 @@ pub(crate) fn get_parquet_stat_min_as_datum( Some(Datum::time_micros(*val)?) } - (PrimitiveType::Timestamp, Statistics::Int64(stats)) => { - stats.min_opt().map(|val| Datum::timestamp_micros(*val)) - } - (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => { - stats.min_opt().map(|val| Datum::timestamptz_micros(*val)) - } + (PrimitiveType::Timestamp, Statistics::Int64(stats)) => stats + .min_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamp_micros), + (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => stats + .min_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamptz_micros), (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => { stats.min_opt().map(|val| Datum::timestamp_nanos(*val)) } @@ -1214,6 +1240,7 @@ pub(crate) fn get_parquet_stat_min_as_datum( pub(crate) fn get_parquet_stat_max_as_datum( primitive_type: &PrimitiveType, stats: &Statistics, + timestamp_unit: Option, ) -> Result> { Ok(match (primitive_type, stats) { (PrimitiveType::Boolean, Statistics::Boolean(stats)) => { @@ -1235,12 +1262,14 @@ pub(crate) fn get_parquet_stat_max_as_datum( Some(Datum::time_micros(*val)?) } - (PrimitiveType::Timestamp, Statistics::Int64(stats)) => { - stats.max_opt().map(|val| Datum::timestamp_micros(*val)) - } - (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => { - stats.max_opt().map(|val| Datum::timestamptz_micros(*val)) - } + (PrimitiveType::Timestamp, Statistics::Int64(stats)) => stats + .max_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamp_micros), + (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => stats + .max_opt() + .and_then(|val| parquet_timestamp_to_micros(*val, timestamp_unit)) + .map(Datum::timestamptz_micros), (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => { stats.max_opt().map(|val| Datum::timestamp_nanos(*val)) } @@ -1623,6 +1652,8 @@ mod tests { use arrow_schema::extension::ExtensionType; use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; use parquet_geospatial::WkbEdges; use super::*; @@ -2776,16 +2807,101 @@ mod tests { PrimitiveType::Geography(GeographyType::default()), ] { assert_eq!( - get_parquet_stat_min_as_datum(&primitive_type, &stats).unwrap(), + get_parquet_stat_min_as_datum(&primitive_type, &stats, None).unwrap(), None ); assert_eq!( - get_parquet_stat_max_as_datum(&primitive_type, &stats).unwrap(), + get_parquet_stat_max_as_datum(&primitive_type, &stats, None).unwrap(), None ); } } + #[test] + fn test_parquet_timestamp_unit() { + let schema = SchemaDescriptor::new(Arc::new( + parse_message_type( + " + message schema { + required int64 utc (TIMESTAMP(MILLIS,true)); + required int64 local (TIMESTAMP(MILLIS,false)); + required int64 legacy (TIMESTAMP_MILLIS); + required int64 micros (TIMESTAMP(MICROS,true)); + required int64 nanos (TIMESTAMP(NANOS,false)); + required int64 legacy_micros (TIMESTAMP_MICROS); + required int64 plain; + } + ", + ) + .unwrap(), + )); + + let units: Vec<_> = schema + .columns() + .iter() + .map(|column| parquet_timestamp_unit(column)) + .collect(); + + assert_eq!(units, vec![ + Some(TimeUnit::Millisecond), + Some(TimeUnit::Millisecond), + Some(TimeUnit::Millisecond), + Some(TimeUnit::Microsecond), + Some(TimeUnit::Nanosecond), + Some(TimeUnit::Microsecond), + None, + ]); + } + + #[test] + fn test_parquet_timestamp_stats_as_micros() { + let bounds = |primitive_type: PrimitiveType, min: i64, max: i64, unit: TimeUnit| { + let stats = Statistics::int64(Some(min), Some(max), None, Some(0), false); + ( + get_parquet_stat_min_as_datum(&primitive_type, &stats, Some(unit)).unwrap(), + get_parquet_stat_max_as_datum(&primitive_type, &stats, Some(unit)).unwrap(), + ) + }; + + assert_eq!( + bounds( + PrimitiveType::Timestamp, + -1_500, + 2_000, + TimeUnit::Millisecond + ), + ( + Some(Datum::timestamp_micros(-1_500_000)), + Some(Datum::timestamp_micros(2_000_000)) + ) + ); + assert_eq!( + bounds( + PrimitiveType::Timestamptz, + -1_500, + 2_000, + TimeUnit::Millisecond + ), + ( + Some(Datum::timestamptz_micros(-1_500_000)), + Some(Datum::timestamptz_micros(2_000_000)) + ) + ); + assert_eq!( + bounds(PrimitiveType::Timestamp, 0, i64::MAX, TimeUnit::Millisecond), + (Some(Datum::timestamp_micros(0)), None) + ); + assert_eq!( + bounds( + PrimitiveType::Timestamp, + -1_500, + 2_000, + TimeUnit::Nanosecond + ), + (None, None) + ); + } + #[test] fn test_type_conversion() { // test primitive type @@ -3235,8 +3351,8 @@ mod tests { scale: 2, }; Ok(( - get_parquet_stat_min_as_datum(&ty, stats)?, - get_parquet_stat_max_as_datum(&ty, stats)?, + get_parquet_stat_min_as_datum(&ty, stats, None)?, + get_parquet_stat_max_as_datum(&ty, stats, None)?, )) } diff --git a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs index 3dfc8ec4a9..9228f169c6 100644 --- a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs @@ -20,6 +20,7 @@ use std::collections::{HashMap, HashSet}; +use arrow_schema::TimeUnit; use fnv::FnvHashSet; use parquet::basic::Type as PhysicalType; use parquet::bloom_filter::Sbbf; @@ -43,6 +44,7 @@ pub(crate) struct ColumnBloomFilter { /// `type_length` from the file's column descriptor. Only meaningful for /// `FIXED_LEN_BYTE_ARRAY`. type_length: i32, + timestamp_unit: Option, } impl ColumnBloomFilter { @@ -51,8 +53,14 @@ impl ColumnBloomFilter { sbbf, physical_type, type_length, + timestamp_unit: None, } } + + pub(crate) fn with_timestamp_unit(mut self, timestamp_unit: Option) -> Self { + self.timestamp_unit = timestamp_unit; + self + } } pub(crate) struct BloomFilterEvaluator<'a> { @@ -251,6 +259,7 @@ fn check_in_bloom_filter(column: &ColumnBloomFilter, datum: &Datum) -> bool { sbbf, physical_type, type_length, + timestamp_unit, } = column; let physical_type = *physical_type; @@ -269,7 +278,17 @@ fn check_in_bloom_filter(column: &ColumnBloomFilter, datum: &Datum) -> bool { _ => true, }, PrimitiveLiteral::Long(v) => match physical_type { - PhysicalType::INT64 => sbbf.check(v), + PhysicalType::INT64 => match (datum.data_type(), timestamp_unit) { + ( + PrimitiveType::Timestamp | PrimitiveType::Timestamptz, + Some(TimeUnit::Millisecond), + ) => v % 1_000 == 0 && sbbf.check(&(v / 1_000)), + ( + PrimitiveType::Timestamp | PrimitiveType::Timestamptz, + Some(TimeUnit::Nanosecond), + ) => true, + _ => sbbf.check(v), + }, PhysicalType::INT32 => match i32::try_from(*v) { Ok(narrowed) => sbbf.check(&narrowed), // Too wide for an INT32 column to hold, so the value is provably @@ -513,7 +532,9 @@ impl BoundPredicateVisitor for BloomFilterEvaluator<'_> { mod tests { use std::collections::HashMap; use std::ops::Not; + use std::sync::Arc; + use arrow_schema::TimeUnit; use parquet::basic::Type as PhysicalType; use parquet::bloom_filter::Sbbf; use parquet::data_type::ByteArray; @@ -1612,4 +1633,40 @@ mod tests { let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); assert!(result, "non-representable double must not prune"); } + + #[test] + fn test_eq_probes_timestamp_in_column_unit() { + let schema = Arc::new(single_field_schema("ts", PrimitiveType::Timestamp)); + let filter = |unit, values: &[i64]| { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(v); + } + HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0) + .with_timestamp_unit(Some(unit)), + )]) + }; + let millis = filter(TimeUnit::Millisecond, &[-1_500, 2_000]); + let nanos = filter(TimeUnit::Nanosecond, &[2_000_000_000]); + + for (filters, micros, expected) in [ + (&millis, 2_000_000, true), + (&millis, 2_000_500, false), + (&millis, -1_500_000, true), + (&millis, 3_000_000, false), + (&nanos, 2_000_000, true), + ] { + let predicate = Reference::new("ts") + .equal_to(Datum::timestamp_micros(micros)) + .bind(schema.clone(), true) + .unwrap(); + assert_eq!( + BloomFilterEvaluator::eval(&predicate, filters).unwrap(), + expected, + "ts = {micros}" + ); + } + } } diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index 50276e26f9..7a6dbc1099 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; +use arrow_schema::TimeUnit; use fnv::FnvHashSet; use ordered_float::OrderedFloat; use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; @@ -27,6 +28,7 @@ use parquet::file::metadata::page_index::RowGroupPageIndex; use parquet::file::page_index::column_index::ColumnIndexMetaData; use parquet::file::page_index::offset_index::OffsetIndexMetaData; +use crate::arrow::{parquet_timestamp_to_micros, parquet_timestamp_unit}; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::decimal_utils::i128_from_be_bytes; @@ -162,16 +164,20 @@ impl<'a> PageIndexEvaluator<'a> { return self.select_all_rows(); }; - // Declared byte width of a FIXED_LEN_BYTE_ARRAY column, used to detect - // truncated page-index bounds when decoding decimals. `None` when the - // column metadata is absent or reports no fixed width. - let type_length = self + let column_descr = self .row_group_metadata .columns() .get(parquet_column_index) - .map(|column| column.column_descr().type_length()) + .map(|column| column.column_descr()); + + // Declared byte width of a FIXED_LEN_BYTE_ARRAY column, used to detect + // truncated page-index bounds when decoding decimals. `None` when the + // column metadata is absent or reports no fixed width. + let type_length = column_descr + .map(|descr| descr.type_length()) .filter(|&len| len > 0) .map(|len| len as usize); + let timestamp_unit = column_descr.and_then(parquet_timestamp_unit); let row_counts = { // Caches row count calculations for columns that appear multiple times in @@ -201,6 +207,7 @@ impl<'a> PageIndexEvaluator<'a> { predicate, field_type, type_length, + timestamp_unit, column_index, &row_counts, )? @@ -249,6 +256,7 @@ impl<'a> PageIndexEvaluator<'a> { predicate: F, field_type: &PrimitiveType, type_length: Option, + timestamp_unit: Option, column_index: &ColumnIndexMetaData, row_counts: &[usize], ) -> Result>> @@ -293,8 +301,12 @@ impl<'a> PageIndexEvaluator<'a> { .zip(row_counts.iter()) .map(|((i, (min, max)), &row_count)| { predicate( - min.map(|&val| Self::int64_bound_to_datum(field_type, val)), - max.map(|&val| Self::int64_bound_to_datum(field_type, val)), + min.and_then(|&val| { + Self::int64_bound_to_datum(field_type, timestamp_unit, val) + }), + max.and_then(|&val| { + Self::int64_bound_to_datum(field_type, timestamp_unit, val) + }), PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) @@ -495,13 +507,24 @@ impl<'a> PageIndexEvaluator<'a> { /// Converts an `INT64` page bound into a [`Datum`] according to the field's /// primitive type. - fn int64_bound_to_datum(field_type: &PrimitiveType, val: i64) -> Datum { + /// + /// Returns `None` for nanosecond bounds under a microsecond field, or when a + /// millisecond bound overflows, so that side of the page cannot be pruned. + fn int64_bound_to_datum( + field_type: &PrimitiveType, + timestamp_unit: Option, + val: i64, + ) -> Option { match field_type { - PrimitiveType::Decimal { .. } => Datum::new( + PrimitiveType::Decimal { .. } => Some(Datum::new( field_type.clone(), PrimitiveLiteral::Int128(i128::from(val)), - ), - _ => Datum::new(field_type.clone(), PrimitiveLiteral::Long(val)), + )), + PrimitiveType::Timestamp | PrimitiveType::Timestamptz => { + parquet_timestamp_to_micros(val, timestamp_unit) + .map(|val| Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))) + } + _ => Some(Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))), } } @@ -968,9 +991,9 @@ mod tests { use arrow_array::{ ArrayRef, Decimal128Array, FixedSizeBinaryArray, Float32Array, Int32Array, - LargeBinaryArray, RecordBatch, StringArray, UInt32Array, + LargeBinaryArray, RecordBatch, StringArray, TimestampMillisecondArray, UInt32Array, }; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::{ ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelector, @@ -1302,6 +1325,45 @@ mod tests { Ok((metadata, temp_file)) } + fn create_millisecond_timestamp_parquet_file( + pages: &[i64], + ) -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + for page in pages { + let array = Arc::new(TimestampMillisecondArray::from_iter_values( + std::iter::repeat_n(*page, 1024), + )) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + /// Creates a single-column parquet file with one 1024-row page per value, /// using `to_array` to build each page. fn create_parquet_file_from_values( @@ -2337,6 +2399,50 @@ mod tests { Ok(()) } + #[test] + fn eval_eq_prunes_millisecond_timestamp_pages() -> Result<()> { + let (metadata, _temp_file) = + create_millisecond_timestamp_parquet_file(&[-1_500, 1_000, 2_000])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::required( + 1, + "ts", + Type::Primitive(PrimitiveType::Timestamp), + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + for (micros, expected) in [ + (2_000_000, vec![ + RowSelector::skip(2048), + RowSelector::select(1024), + ]), + (-1_500_000, vec![ + RowSelector::select(1024), + RowSelector::skip(2048), + ]), + ] { + let filter = Reference::new("ts") + .equal_to(Datum::timestamp_micros(micros)) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, expected, "ts = {micros}"); + } + + Ok(()) + } + #[test] fn eval_keeps_int32_pages_of_promoted_long_field() -> Result<()> { // An `int` -> `long` promotion leaves older files with INT32 page diff --git a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs index ea3e135959..11b76bb0ba 100644 --- a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs @@ -19,11 +19,14 @@ use std::collections::HashMap; +use arrow_schema::TimeUnit; use fnv::FnvHashSet; use parquet::file::metadata::RowGroupMetaData; use parquet::file::statistics::Statistics; -use crate::arrow::{get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum}; +use crate::arrow::{ + get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, parquet_timestamp_unit, +}; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, Schema}; @@ -80,6 +83,17 @@ impl<'a> RowGroupMetricsEvaluator<'a> { .statistics() } + fn timestamp_unit(&self, field_id: i32) -> Option { + let parquet_column_index = *self + .iceberg_field_id_to_parquet_column_index + .get(&field_id)?; + parquet_timestamp_unit( + self.row_group_metadata + .column(parquet_column_index) + .column_descr(), + ) + } + fn null_count(&self, field_id: i32) -> Option { self.stats_for_field_id(field_id)?.null_count_opt() } @@ -140,7 +154,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { return Ok(None); }; - get_parquet_stat_min_as_datum(&primitive_type, stats) + get_parquet_stat_min_as_datum(&primitive_type, stats, self.timestamp_unit(field_id)) } fn max_value(&self, field_id: i32) -> Result> { @@ -148,7 +162,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { return Ok(None); }; - get_parquet_stat_max_as_datum(&primitive_type, stats) + get_parquet_stat_max_as_datum(&primitive_type, stats, self.timestamp_unit(field_id)) } fn visit_inequality( @@ -310,11 +324,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_CANT_MATCH; } - let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { - return ROW_GROUP_MIGHT_MATCH; - }; - - if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { + if let Some(lower_bound) = self.min_value(field_id)? { if lower_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -324,7 +334,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { + if let Some(upper_bound) = self.max_value(field_id)? { if upper_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -368,11 +378,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { )); }; - let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { - return ROW_GROUP_MIGHT_MATCH; - }; - - if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { + if let Some(lower_bound) = self.min_value(field_id)? { let PrimitiveLiteral::String(lower_bound) = lower_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -390,7 +396,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { + if let Some(upper_bound) = self.max_value(field_id)? { let PrimitiveLiteral::String(upper_bound) = upper_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -497,11 +503,8 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_MIGHT_MATCH; } - let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { - return ROW_GROUP_MIGHT_MATCH; - }; - let lower_bound = get_parquet_stat_min_as_datum(&primitive_type, stats)?; - let upper_bound = get_parquet_stat_max_as_datum(&primitive_type, stats)?; + let lower_bound = self.min_value(field_id)?; + let upper_bound = self.max_value(field_id)?; // A NaN bound is unreliable on that side only. Drop it to unbounded // so a valid bound can still prune. See InclusiveMetricsEvaluator. diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index d1a021d8e1..73a19db84b 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -38,7 +38,7 @@ use parquet_geospatial::interval::{Interval, IntervalTrait, WraparoundInterval}; use super::{FileWriter, FileWriterBuilder}; use crate::arrow::{ ArrowFileReader, DEFAULT_MAP_FIELD_NAME, FieldMatchMode, NanValueCountVisitor, - get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, + get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, parquet_timestamp_unit, }; use crate::compression::CompressionCodec; use crate::encryption::{EncryptionManager, StandardKeyMetadata}; @@ -483,9 +483,10 @@ impl MinMaxColAggregator { return Ok(()); }; let ty = ty.clone(); + let timestamp_unit = parquet_timestamp_unit(column.column_descr()); if value.min_is_exact() { - let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value)? else { + let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value, timestamp_unit)? else { return Err(Error::new( ErrorKind::Unexpected, format!("Statistics {value} is not match with field type {ty}."), @@ -499,7 +500,7 @@ impl MinMaxColAggregator { } if value.max_is_exact() { - let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value)? else { + let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value, timestamp_unit)? else { return Err(Error::new( ErrorKind::Unexpected, format!("Statistics {value} is not match with field type {ty}."), @@ -878,13 +879,13 @@ mod tests { use arrow_array::types::{Float32Type, Int64Type}; use arrow_array::{ Array, ArrayRef, BooleanArray, Decimal128Array, Float32Array, Float64Array, Int32Array, - Int64Array, ListArray, MapArray, RecordBatch, StructArray, + Int64Array, ListArray, MapArray, RecordBatch, StructArray, TimestampMillisecondArray, }; - use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef}; + use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef, TimeUnit}; use arrow_select::concat::concat_batches; use futures::TryStreamExt; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; use parquet::basic::{ BrotliLevel, Compression, EdgeInterpolationAlgorithm, GzipLevel, LogicalType, ZstdLevel, }; @@ -3162,6 +3163,58 @@ mod tests { assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))])); } + #[test] + fn test_parquet_to_data_file_scales_millisecond_timestamp_bounds() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + TimestampMillisecondArray::from(vec![-1_500, 2_000]), + )]) + .unwrap(); + let mut buf = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut buf, arrow_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let metadata = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(buf)) + .unwrap() + .metadata() + .clone(); + + let data_file = ParquetWriter::parquet_to_data_file_builder( + schema, + metadata, + 0, + "ts.parquet".to_string(), + HashMap::new(), + None, + ) + .unwrap() + .partition_spec_id(0) + .build() + .unwrap(); + + assert_eq!( + data_file.lower_bounds(), + &HashMap::from([(1, Datum::timestamp_micros(-1_500_000))]) + ); + assert_eq!( + data_file.upper_bounds(), + &HashMap::from([(1, Datum::timestamp_micros(2_000_000))]) + ); + } + fn cdc_test_schema() -> SchemaRef { Arc::new( Schema::builder()